From 2afab9a63b9a9dcc564c5c378ecab3720c721c2c Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Mon, 12 Oct 2026 00:43:29 +0200 Subject: [PATCH 01/98] build: do not include shared gtest in `node` link path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antoine du Hamel PR-URL: https://github.com/nodejs/node/pull/66626 Reviewed-By: Ulises Gascón Reviewed-By: Richard Lau --- configure.py | 15 +++++++++------ node.gyp | 3 ++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/configure.py b/configure.py index acda6a16c96c..e38dd91cdb9d 100755 --- a/configure.py +++ b/configure.py @@ -2195,9 +2195,10 @@ def configure_napi(output): version = getnapibuildversion.get_napi_version() output['variables']['napi_build_version'] = version -def configure_library(lib, output, pkgname=None): +def configure_library(lib, output, pkgname=None, libraries_key='libraries'): shared_lib = 'shared_' + lib output['variables']['node_' + shared_lib] = b(getattr(options, shared_lib)) + libraries = output[libraries_key] if getattr(options, shared_lib): (pkg_libs, pkg_cflags, pkg_libpath, _) = pkg_config(pkgname or lib) @@ -2216,17 +2217,17 @@ def configure_library(lib, output, pkgname=None): output['msvs_settings']['VCLinkerTool']['AdditionalOptions'] += [ f"/LIBPATH:{options.__dict__[shared_lib + '_libpath']}"] else: - output['libraries'] += [ + libraries += [ f"-L{options.__dict__[shared_lib + '_libpath']}"] elif pkg_libpath: - output['libraries'] += [pkg_libpath] + libraries += [pkg_libpath] default_libs = getattr(options, shared_lib + '_libname') if default_libs: - output['libraries'] += [f'-l{l}' for l in default_libs.split(',')] + libraries += [f'-l{l}' for l in default_libs.split(',')] elif pkg_libs: - output['libraries'] += pkg_libs.split() + libraries += pkg_libs.split() def configure_v8(o, configs): @@ -2974,7 +2975,9 @@ def make_bin_override(): configure_library('simdutf', output) configure_library('brotli', output, pkgname=['libbrotlidec', 'libbrotlienc']) configure_library('cares', output, pkgname='libcares') -configure_library('gtest', output) +# Only cctest uses gtest; keep it out of the node binary's link line. +output['variables']['node_shared_gtest_libraries'] = [] +configure_library('gtest', output, 'node_shared_gtest_libraries') configure_library('hdr_histogram', output) configure_library('highway', output, pkgname='libhwy') configure_library('merve', output) diff --git a/node.gyp b/node.gyp index 587df5cace14..5334b8a746da 100644 --- a/node.gyp +++ b/node.gyp @@ -19,6 +19,7 @@ 'node_shared_brotli%': 'false', 'node_shared_cares%': 'false', 'node_shared_gtest%': 'false', + 'node_shared_gtest_libraries%': [], 'node_shared_hdr_histogram%': 'false', 'node_shared_highway%': 'false', 'node_shared_http_parser%': 'false', @@ -1373,7 +1374,7 @@ ], }], [ 'node_shared_gtest=="true"', { - 'libraries': [ '-lgtest_main' ], + 'libraries': [ '-lgtest_main', '<@(node_shared_gtest_libraries)' ], }], [ 'node_use_bundled_v8!="false" and node_shared_abseil=="false"', { 'dependencies': [ 'tools/v8_gypfiles/abseil.gyp:abseil' ], From 1a33e914e8e1ae488901ddc50718aa0357846f2f Mon Sep 17 00:00:00 2001 From: inoway46 Date: Wed, 7 Oct 2026 16:07:36 +0900 Subject: [PATCH 02/98] deps: V8: cherry-pick 7d5eb1df2a72 Backport the Maglev HeapNumber materialization fix to V8 14.6. Use the existing offset constants and virtual-object slot traversal. Original commit message: [maglev] Materialize a fresh HeapNumber per deoptimization This matches what Turbolev graph builder already does to Maglev. Fixed: 547819997 Change-Id: Iafc1185ef96db8816db36b295a3a4b7e528f9863 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/8255554 Reviewed-by: Leszek Swirski Commit-Queue: Victor Gomes Cr-Commit-Position: refs/heads/main@{#109347} Refs: https://github.com/v8/v8/commit/7d5eb1df2a721c8fe3aa03b999464c4b86242080 Refs: https://github.com/nodejs/node/issues/66457 Refs: https://www.cve.org/CVERecord?id=CVE-2026-85045 Signed-off-by: inoway46 Assisted-by: Codex PR-URL: https://github.com/nodejs/node/pull/66566 Reviewed-By: Richard Lau Reviewed-By: Antoine du Hamel --- common.gypi | 2 +- deps/v8/src/maglev/maglev-code-generator.cc | 30 +++++++-------- deps/v8/src/maglev/maglev-ir.h | 12 +++--- .../test/mjsunit/maglev/regress-547819997.js | 38 +++++++++++++++++++ 4 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 deps/v8/test/mjsunit/maglev/regress-547819997.js diff --git a/common.gypi b/common.gypi index de80a68f492f..eeebab06807f 100644 --- a/common.gypi +++ b/common.gypi @@ -44,7 +44,7 @@ # Reset this number to 0 on major V8 upgrades. # Increment by one for each non-official patch applied to deps/v8. - 'v8_embedder_string': '-node.37', + 'v8_embedder_string': '-node.38', ##### V8 defaults for Node.js ##### diff --git a/deps/v8/src/maglev/maglev-code-generator.cc b/deps/v8/src/maglev/maglev-code-generator.cc index 5cd614bd3bf2..ebea2905d39c 100644 --- a/deps/v8/src/maglev/maglev-code-generator.cc +++ b/deps/v8/src/maglev/maglev-code-generator.cc @@ -1499,17 +1499,9 @@ class MaglevFrameTranslationBuilder { return kNotDuplicated; } - void BuildHeapNumber(const VirtualObject* vobject) { - DCHECK_EQ(vobject->object_type(), vobj::ObjectType::kHeapNumber); - ValueNode* value_node = vobject->get(HeapNumber::kValueOffset); - return BuildHeapNumber(value_node->Cast()->value()); - } - - void BuildHeapNumber(Float64 number) { - DirectHandle value = - local_isolate_->factory()->NewHeapNumberFromBits( - number.get_bits()); - translation_array_builder_->StoreLiteral(GetDeoptLiteral(*value)); + int CreateUnduplicatableId() { + object_ids_.push_back(kNotDuplicated); + return kNotDuplicated; } void BuildNestedValue(const ValueNode* value, @@ -1564,12 +1556,16 @@ class MaglevFrameTranslationBuilder { const InputLocation*& input_location, const VirtualObjectList& virtual_objects) { vobj::ObjectType object_type = object->object_type(); - if (object_type == vobj::ObjectType::kHeapNumber) { - // TODO(jgruber): Could we use the standard path below instead? - return BuildHeapNumber(object); - } + DCHECK_NOT_NULL(object->allocation()); + // HeapNumbers may be mutable object fields; each materialization must + // create a fresh box, so they are never deduplicated. + // TODO(victorgomes): Constrain which objects may contain mutable + // HeapNumbers. Immutable HeapNumbers can be stored as a literal object + // instead of a captured object. int dup_id = - GetDuplicatedId(reinterpret_cast(object->allocation())); + object_type == vobj::ObjectType::kHeapNumber + ? CreateUnduplicatableId() + : GetDuplicatedId(reinterpret_cast(object->allocation())); if (dup_id != kNotDuplicated) { translation_array_builder_->DuplicateObject(dup_id); object->ForEachNestedRuntimeInput( @@ -1708,7 +1704,7 @@ class MaglevFrameTranslationBuilder { IdentityMap* protected_deopt_literals_; IdentityMap* deopt_literals_; - static const int kNotDuplicated = -1; + static constexpr int kNotDuplicated = -1; std::vector object_ids_; }; diff --git a/deps/v8/src/maglev/maglev-ir.h b/deps/v8/src/maglev/maglev-ir.h index d6282a00ebb3..d3720049d225 100644 --- a/deps/v8/src/maglev/maglev-ir.h +++ b/deps/v8/src/maglev/maglev-ir.h @@ -5622,11 +5622,9 @@ class VirtualObject : public FixedInputValueNodeT<0, VirtualObject> { vobj::Field snd = FieldForOffset(ConsString::kSecondOffset); return callback(slots_[snd.slot_index], snd); } - if (object_type() == vobj::ObjectType::kHeapNumber) { - // HeapNumber materialization creates a literal object instead of - // slot traversal. - return true; - } + // TODO(victorgomes): Constrain which objects may contain mutable + // HeapNumbers. Immutable HeapNumbers can be stored as a literal object + // instead of traversing their slots. } for (int i = 0; i < slot_count(); i++) { if (!callback(slots_[i], FieldForSlot(i))) { @@ -5917,8 +5915,8 @@ struct VirtualPrimitiveHeapObjectShape : VirtualHeapObjectShape {}; struct VirtualHeapNumberShape : VirtualPrimitiveHeapObjectShape { using T = HeapNumber; - // Special handling needed; deopt materialization uses a special path. - // TODO(jgruber): .. but could it take the standard path instead? + // Special handling needed; instances may be mutable object fields and thus + // must never be deduplicated in deopt frames. static constexpr vobj::ObjectType kObjectType = vobj::ObjectType::kHeapNumber; #define FIELD_LIST(V) V(value, T::kValueOffset, vobj::FieldType::kFloat64) DEF_SHAPE(VirtualPrimitiveHeapObjectShape, FIELD_LIST); diff --git a/deps/v8/test/mjsunit/maglev/regress-547819997.js b/deps/v8/test/mjsunit/maglev/regress-547819997.js new file mode 100644 index 000000000000..43a8e395d3fc --- /dev/null +++ b/deps/v8/test/mjsunit/maglev/regress-547819997.js @@ -0,0 +1,38 @@ +// Copyright 2026 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Flags: --allow-natives-syntax --maglev + +const boxes = []; + +// Never called during warm-up, so this call site has no feedback and Maglev +// emits an unconditional deopt for it. The literal below is therefore only +// used by deopt frames and gets elided by escape analysis. +function sink(o) { + boxes.push(o); +} + +function foo(depth, take) { + const o = {x: 1.5}; + if (depth > 0) foo(depth - 1, take); + if (take) sink(o); +} + +%PrepareFunctionForOptimization(foo); +foo(3, false); +foo(3, false); +%OptimizeMaglevOnNextCall(foo); +foo(3, false); + +// Deopts the innermost activation eagerly and the outer ones lazily, so every +// activation materializes its own object. +foo(3, true); + +assertEquals(4, boxes.length); +for (let i = 0; i < boxes.length; i++) { + boxes[i].x = i; +} +for (let i = 0; i < boxes.length; i++) { + assertEquals(i, boxes[i].x); +} From d4c87eb086f33157a04ed7eba4cee73f2b8617df Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sat, 26 Sep 2026 13:15:24 +0200 Subject: [PATCH 03/98] test: scope Blob WPT failures to big-endian The typed array Blob constructor tests expect little-endian byte order. Mark them as expected failures only on big-endian platforms, so failures on little-endian platforms remain visible. Signed-off-by: Filip Skokan Assisted-by: Claude, Codex PR-URL: https://github.com/nodejs/node/pull/66322 Reviewed-By: Daeyeon Jeong --- test/wpt/status/FileAPI/blob.cjs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/wpt/status/FileAPI/blob.cjs b/test/wpt/status/FileAPI/blob.cjs index 6f8599dc8b4e..c59516ca1dc5 100644 --- a/test/wpt/status/FileAPI/blob.cjs +++ b/test/wpt/status/FileAPI/blob.cjs @@ -23,14 +23,17 @@ module.exports = { 'Blob-constructor-dom.window.js': { skip: 'Depends on DOM API', }, - 'Blob-constructor.any.js': { - fail: { - flaky: [ - 'Passing typed arrays as elements of the blobParts array should work.', - 'Passing a Float16Array as element of the blobParts array should work.', - 'Passing a Float64Array as element of the blobParts array should work.', - 'Passing BigInt typed arrays as elements of the blobParts array should work.', - ], + ...(os.endianness() === 'BE' ? { + 'Blob-constructor.any.js': { + fail: { + note: 'The expected bytes assume a little-endian platform', + expected: [ + 'Passing typed arrays as elements of the blobParts array should work.', + 'Passing a Float16Array as element of the blobParts array should work.', + 'Passing a Float64Array as element of the blobParts array should work.', + 'Passing BigInt typed arrays as elements of the blobParts array should work.', + ], + }, }, - }, + } : {}), }; From 7d4fda6ba7de55b8d63d7309f97b94fff07b690b Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sat, 26 Sep 2026 13:22:26 +0200 Subject: [PATCH 04/98] test: honor allowed WPT promise rejections Honor allow_uncaught_exception for unhandled promise rejections in both WPT worker paths. Apply setup settings before nested callbacks, preserve settings after results start, and apply promise_setup settings when its callback runs. Keep rejection failures in single_test mode. Cover both runner backends and Web Workers, and remove the flaky expectation for the Web Locks test that deliberately rejects. Refs: https://github.com/nodejs/node/issues/59142 Signed-off-by: Filip Skokan Assisted-by: Claude, Codex PR-URL: https://github.com/nodejs/node/pull/66322 Reviewed-By: Daeyeon Jeong --- test/common/wpt/rejections.js | 65 ++++++ test/common/wpt/webworker.js | 2 + test/common/wpt/worker.js | 2 + .../test-common-wpt-allowed-rejections.js | 189 ++++++++++++++++++ test/wpt/status/web-locks.json | 8 - 5 files changed, 258 insertions(+), 8 deletions(-) create mode 100644 test/common/wpt/rejections.js create mode 100644 test/parallel/test-common-wpt-allowed-rejections.js diff --git a/test/common/wpt/rejections.js b/test/common/wpt/rejections.js new file mode 100644 index 000000000000..f0cda7a4c603 --- /dev/null +++ b/test/common/wpt/rejections.js @@ -0,0 +1,65 @@ +'use strict'; + +// Node.js does not dispatch the global unhandledrejection event used by +// testharness.js. Ignore rejections only when the harness settings allow them. +module.exports = function honorAllowedRejections() { + const { setup, promise_setup, add_result_callback, add_completion_callback } = globalThis; + let configurable = true; + let allowed = false; + let singleTest = false; + const ignoreRejection = () => {}; + + // The harness ignores configuration changes once results have started. + add_result_callback(() => { configurable = false; }); + add_completion_callback(() => { configurable = false; }); + + function apply(properties) { + if (properties) { + if (Object.prototype.propertyIsEnumerable.call(properties, 'allow_uncaught_exception')) { + allowed = properties.allow_uncaught_exception; + } + if (Object.prototype.propertyIsEnumerable.call(properties, 'single_test') && properties.single_test) { + singleTest = true; + } + } + process.removeListener('unhandledRejection', ignoreRejection); + // In single_test mode the harness still fails the implicit test when + // uncaught errors are allowed. Retain Node's failure path for that mode. + if (allowed && !singleTest) { + process.on('unhandledRejection', ignoreRejection); + } + } + + globalThis.setup = function(funcOrProperties, maybeProperties) { + let func; + let properties; + if (arguments.length === 2) { + func = funcOrProperties; + properties = maybeProperties; + } else if (funcOrProperties instanceof Function) { + func = funcOrProperties; + properties = {}; + } else { + properties = funcOrProperties; + } + + // This callback runs only when setup was accepted, before any nested + // setup calls made by the test's callback can override its settings. + return setup.call(this, () => { + apply(properties); + if (func) func(); + }, properties); + }; + + globalThis.promise_setup = function(func, properties) { + if (typeof func !== 'function') { + return promise_setup.call(this, func, properties); + } + return promise_setup.call(this, () => { + // promise_setup applies settings when its queued callback executes. + // Unlike setup, it still invokes that callback after results start. + if (configurable) apply(properties); + return func(); + }, properties); + }; +}; diff --git a/test/common/wpt/webworker.js b/test/common/wpt/webworker.js index e9a8ebe53d79..8a335c808692 100644 --- a/test/common/wpt/webworker.js +++ b/test/common/wpt/webworker.js @@ -4,6 +4,7 @@ // Refs: https://web-platform-tests.org/writing-tests/testharness.html const { pathToFileURL } = require('url'); +const honorAllowedRejections = require('./rejections'); const { runInThisContext, constants: { USE_MAIN_CONTEXT_DEFAULT_LOADER }, @@ -88,6 +89,7 @@ globalThis.onmessage = ({ data }) => { const result = realImportScripts.apply(this, mapped); if (mapped.includes(testharnessPath)) { applySkips(); + honorAllowedRejections(); } return result; }; diff --git a/test/common/wpt/worker.js b/test/common/wpt/worker.js index ca4f4706f89b..7d84f2ac1a36 100644 --- a/test/common/wpt/worker.js +++ b/test/common/wpt/worker.js @@ -9,6 +9,7 @@ const { } = require('vm'); const { setFlagsFromString } = require('v8'); const { inspect } = require('util'); +const honorAllowedRejections = require('./rejections'); const { isMainThread, parentPort, @@ -79,6 +80,7 @@ function run(workerData) { filename: workerData.harness.filename, importModuleDynamically: USE_MAIN_CONTEXT_DEFAULT_LOADER, }); + honorAllowedRejections(); // If there are skip patterns, wrap test functions to prevent execution of // matching tests. This must happen after testharness.js is loaded but before diff --git a/test/parallel/test-common-wpt-allowed-rejections.js b/test/parallel/test-common-wpt-allowed-rejections.js new file mode 100644 index 000000000000..e8f8ed113f3f --- /dev/null +++ b/test/parallel/test-common-wpt-allowed-rejections.js @@ -0,0 +1,189 @@ +'use strict'; + +const common = require('../common'); +const fixtures = require('../common/fixtures'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { backends } = require('../common/wpt'); + +const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js'); +const harness = { + code: fs.readFileSync(harnessPath, 'utf8'), + filename: harnessPath, +}; + +async function check(backend, webWorker, { + name, + code, + allowed, + passes = 1, + singleTest = false, + message = /uncaught rejection probe/, +}) { + const label = `${backend}, ${webWorker}, ${name}`; + const script = { + filename: fixtures.path('wpt-rejection-probe.js'), + // Use a direct worker script for single_test: .any.js wrappers call + // done() automatically, completing the implicit test before its timers. + code: `${webWorker && singleTest ? 'importScripts("/resources/testharness.js");' : ''}\n${code}`, + }; + const workerData = { + testRelativePath: 'rejection-probe.any.js', + wptRunner: path.join(__dirname, '../common/wpt.js'), + wptPath: 'compression', + harness, + scriptsToRun: webWorker ? [] : [script], + webWorker: webWorker ? { + path: script.filename, + modifiedScript: script, + isAnyTest: !singleTest, + variant: '', + scripts: [], + } : undefined, + }; + + let result; + const statuses = []; + const handle = backends[backend](['--experimental-web-worker'], workerData, { + message(message) { + if (message.type === 'result') { + statuses.push(message.result.status); + } else if (message.type === 'completion') { + // The process backend can complete before its uncaught-error handler + // exits. Preserve any failure it already reported. + result ||= message.status; + handle.kill(); + } else { + assert.fail(`Unexpected message type: ${message.type}`); + } + }, + failure(error) { + if (result) return false; + result = { status: 1, message: error.message }; + return true; + }, + }); + const timeout = setTimeout(() => { + handle.kill(); + assert.fail(`WPT rejection probe did not finish: ${label}`); + }, common.platformTimeout(10_000)); + + try { + await handle.finished; + } finally { + clearTimeout(timeout); + } + + assert(result, label); + assert.strictEqual(result.status, allowed ? 0 : 1, label); + if (allowed) { + assert.deepStrictEqual(statuses, Array(passes).fill(0), label); + } else { + assert.match(result.message, message, label); + } +} + +(async () => { + for (const backend of ['thread', 'process']) { + for (const webWorker of [false, true]) { + for (const [setup, allowed] of [ + ['', false], + ['setup({ allow_uncaught_exception: false });', false], + ['setup({ allow_uncaught_exception: true });', true], + ['setup(() => {}, { allow_uncaught_exception: true });', true], + ['setup(null, { allow_uncaught_exception: true });', true], + ['setup({ allow_uncaught_exception: true }); setup({});', true], + ['setup({ allow_uncaught_exception: true }); setup({ allow_uncaught_exception: false });', false], + ['setup(Object.defineProperty({}, "allow_uncaught_exception", { value: true }));', false], + ]) { + await check(backend, webWorker, { + name: setup || 'default settings', + code: `${setup} + const t = async_test('rejection probe'); + Promise.reject(new Error('uncaught rejection probe')); + setTimeout(() => t.done(), 0);`, + allowed, + }); + } + + for (const allowed of [false, true]) { + await check(backend, webWorker, { + name: `nested setup allows rejection: ${allowed}`, + code: `setup(() => setup({ allow_uncaught_exception: ${allowed} }), + { allow_uncaught_exception: ${!allowed} }); + const t = async_test('rejection probe'); + Promise.reject(new Error('uncaught rejection probe')); + setTimeout(() => t.done(), 0);`, + allowed, + }); + + await check(backend, webWorker, { + name: `late setup preserves allowance: ${allowed}`, + code: `setup({ allow_uncaught_exception: ${allowed} }); + const t = async_test('rejection probe'); + test(() => {}, 'first result'); + setup(() => assert_unreached('late setup callback'), + { allow_uncaught_exception: ${!allowed} }); + Promise.reject(new Error('uncaught rejection probe')); + setTimeout(() => t.done(), 0);`, + allowed, + passes: 2, + }); + + await check(backend, webWorker, { + name: `promise_setup applies deferred allowance: ${allowed}`, + code: `promise_setup(async () => {}, { allow_uncaught_exception: ${allowed} }); + setup({ allow_uncaught_exception: ${!allowed} }); + promise_test(async () => { + Promise.reject(new Error('uncaught rejection probe')); + await new Promise(resolve => setTimeout(resolve, 0)); + }, 'rejection probe');`, + allowed, + }); + + await check(backend, webWorker, { + name: `late promise_setup preserves allowance: ${allowed}`, + code: `setup({ allow_uncaught_exception: ${allowed} }); + const t = async_test('rejection probe'); + test(() => {}, 'first result'); + promise_setup(async () => { + Promise.reject(new Error('uncaught rejection probe')); + setTimeout(() => t.done(), 0); + }, { allow_uncaught_exception: ${!allowed} });`, + allowed, + passes: 2, + }); + + await check(backend, webWorker, { + name: `single_test still fails with allowance: ${allowed}`, + code: `setup({ single_test: true, allow_uncaught_exception: ${allowed} }); + setup({ single_test: false }); + Promise.reject(new Error('uncaught rejection probe')); + setTimeout(done, 0);`, + allowed: false, + singleTest: true, + }); + } + + for (const setup of [ + 'setup(() => { throw new Error("setup failure"); }, { allow_uncaught_exception: true });', + 'promise_setup(async () => { throw new Error("setup failure"); }, { allow_uncaught_exception: true });', + ]) { + await check(backend, webWorker, { + name: setup, + code: `${setup}\npromise_test(async () => {}, 'setup must succeed first');`, + allowed: false, + message: /setup failure/, + }); + } + + await check(backend, webWorker, { + name: 'unexpected rejection fails even with a permanently pending test', + code: `async_test('pending forever'); + Promise.reject(new Error('uncaught rejection probe'));`, + allowed: false, + }); + } + } +})().then(common.mustCall()); diff --git a/test/wpt/status/web-locks.json b/test/wpt/status/web-locks.json index 97a0a7900d64..f7774a38fa76 100644 --- a/test/wpt/status/web-locks.json +++ b/test/wpt/status/web-locks.json @@ -1,12 +1,4 @@ { - "held.https.any.js": { - "fail": { - "note": "Flaky on ppc, linux x64 and s390x #59142", - "flaky": [ - "Error: this uncaught rejection is expected" - ] - } - }, "idlharness.https.any.js": { "fail": { "expected": [ From c348a4cb46c7495f6d8eafe12ab1a06171c4872a Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sat, 26 Sep 2026 15:38:13 +0200 Subject: [PATCH 05/98] test: run instantiateStreaming-bad-imports WPT This test was skipped after hangs with V8 11.2. The associated compiler shutdown deadlock involved NodePlatform::DrainTasks waiting for worker tasks that needed garbage collection on the blocked main thread. Since 5fb879c4584, DrainTasks only waits for user-blocking tasks, which excludes ordinary background compilation. Remove the obsolete skip. Refs: https://github.com/nodejs/node/issues/47297 Signed-off-by: Filip Skokan Assisted-by: Claude, Codex PR-URL: https://github.com/nodejs/node/pull/66322 Reviewed-By: Daeyeon Jeong --- test/wpt/status/wasm/webapi.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/wpt/status/wasm/webapi.json b/test/wpt/status/wasm/webapi.json index 3e2075655e5a..6328e55dc18b 100644 --- a/test/wpt/status/wasm/webapi.json +++ b/test/wpt/status/wasm/webapi.json @@ -16,8 +16,5 @@ }, "status.any.js": { "skip": "WPTRunner does not support fetch()" - }, - "instantiateStreaming-bad-imports.any.js": { - "skip": "Flaky on ARM with V8 >= 11.2" } } From 6fbffd5ee58a7aae11b031ea6b87ef83c5a98990 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:34:29 +0000 Subject: [PATCH 06/98] meta: bump codecov/codecov-action from 7.1.0 to 7.1.1 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 7.1.0 to 7.1.1. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/0b35c9ecc4f0529d0eb674914510c22f85b196b4...303a32d7a59b442fa8d48b6a1cc6825c09c847a5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 7.1.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] PR-URL: https://github.com/nodejs/node/pull/66448 Reviewed-By: Colin Ihrig Reviewed-By: Luigi Pinca --- .github/workflows/coverage-linux-without-intl.yml | 2 +- .github/workflows/coverage-linux.yml | 2 +- .github/workflows/coverage-windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage-linux-without-intl.yml b/.github/workflows/coverage-linux-without-intl.yml index 9b8f8e2fd831..aa5ed3a3747f 100644 --- a/.github/workflows/coverage-linux-without-intl.yml +++ b/.github/workflows/coverage-linux-without-intl.yml @@ -85,6 +85,6 @@ jobs: - name: Clean tmp run: rm -rf coverage/tmp && rm -rf out - name: Upload - uses: codecov/codecov-action@0b35c9ecc4f0529d0eb674914510c22f85b196b4 # v7.1.0 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: directory: ./coverage diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 0c7fc292d346..63ac4fa5f2e0 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -85,6 +85,6 @@ jobs: - name: Clean tmp run: rm -rf coverage/tmp && rm -rf out - name: Upload - uses: codecov/codecov-action@0b35c9ecc4f0529d0eb674914510c22f85b196b4 # v7.1.0 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: directory: ./coverage diff --git a/.github/workflows/coverage-windows.yml b/.github/workflows/coverage-windows.yml index a4db4ccc08c7..f30913081973 100644 --- a/.github/workflows/coverage-windows.yml +++ b/.github/workflows/coverage-windows.yml @@ -101,6 +101,6 @@ jobs: - name: Clean tmp run: npx rimraf ./coverage/tmp - name: Upload - uses: codecov/codecov-action@0b35c9ecc4f0529d0eb674914510c22f85b196b4 # v7.1.0 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: directory: ./coverage From 4760c2790477c9cf21939ae2aa2876992f115492 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:34:40 +0000 Subject: [PATCH 07/98] tools: bump the eslint group in /tools/eslint with 5 updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the eslint group in /tools/eslint with 5 updates: | Package | From | To | | --- | --- | --- | | [@babel/core](https://github.com/babel/babel/tree/HEAD/packages/babel-core) | `8.0.5` | `8.0.6` | | [@babel/eslint-parser](https://github.com/babel/babel/tree/HEAD/eslint/babel-eslint-parser) | `8.0.5` | `8.0.6` | | [eslint](https://github.com/eslint/eslint) | `10.10.0` | `10.11.0` | | [eslint-plugin-jsdoc](https://github.com/gajus/eslint-plugin-jsdoc) | `64.3.10` | `64.5.4` | | [eslint-plugin-regexp](https://github.com/ota-meshi/eslint-plugin-regexp) | `3.3.0` | `3.3.1` | Updates `@babel/core` from 8.0.5 to 8.0.6 - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v8.0.6/packages/babel-core) Updates `@babel/eslint-parser` from 8.0.5 to 8.0.6 - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v8.0.6/eslint/babel-eslint-parser) Updates `eslint` from 10.10.0 to 10.11.0 - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.10.0...v10.11.0) Updates `eslint-plugin-jsdoc` from 64.3.10 to 64.5.4 - [Release notes](https://github.com/gajus/eslint-plugin-jsdoc/releases) - [Commits](https://github.com/gajus/eslint-plugin-jsdoc/compare/v64.3.10...v64.5.4) Updates `eslint-plugin-regexp` from 3.3.0 to 3.3.1 - [Release notes](https://github.com/ota-meshi/eslint-plugin-regexp/releases) - [Changelog](https://github.com/ota-meshi/eslint-plugin-regexp/blob/master/CHANGELOG.md) - [Commits](https://github.com/ota-meshi/eslint-plugin-regexp/compare/v3.3.0...v3.3.1) --- updated-dependencies: - dependency-name: "@babel/core" dependency-version: 8.0.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: eslint - dependency-name: "@babel/eslint-parser" dependency-version: 8.0.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: eslint - dependency-name: eslint dependency-version: 10.11.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: eslint - dependency-name: eslint-plugin-jsdoc dependency-version: 64.5.4 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: eslint - dependency-name: eslint-plugin-regexp dependency-version: 3.3.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: eslint ... Signed-off-by: dependabot[bot] PR-URL: https://github.com/nodejs/node/pull/66445 Reviewed-By: Luigi Pinca Reviewed-By: René Reviewed-By: Colin Ihrig --- tools/eslint/package-lock.json | 224 +++++++++++++++++---------------- tools/eslint/package.json | 10 +- 2 files changed, 120 insertions(+), 114 deletions(-) diff --git a/tools/eslint/package-lock.json b/tools/eslint/package-lock.json index e56bfa9872b9..26247efca8b2 100644 --- a/tools/eslint/package-lock.json +++ b/tools/eslint/package-lock.json @@ -8,27 +8,27 @@ "name": "eslint-tools", "version": "0.0.0", "dependencies": { - "@babel/core": "^8.0.5", - "@babel/eslint-parser": "^8.0.5", + "@babel/core": "^8.0.6", + "@babel/eslint-parser": "^8.0.6", "@babel/plugin-syntax-import-defer": "^8.0.1", "@babel/plugin-syntax-import-source": "^8.0.1", "@eslint/js": "^10.0.1", "@eslint/markdown": "^8.0.3", "@stylistic/eslint-plugin": "^5.10.0", - "eslint": "^10.10.0", + "eslint": "^10.11.0", "eslint-formatter-tap": "^9.0.1", - "eslint-plugin-jsdoc": "^64.3.10", - "eslint-plugin-regexp": "^3.3.0", + "eslint-plugin-jsdoc": "^64.5.4", + "eslint-plugin-regexp": "^3.3.1", "globals": "^17.12.0" } }, "node_modules/@babel/code-frame": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-8.0.0.tgz", - "integrity": "sha512-dYYg153EyN2Ekbqw2zAsbd6/JR+9N2SEoC7YV2GyyqMM7x9bLDTjBD6XBhSMLH0wtIVyJj03jWNriQhaN+eoCw==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-8.0.6.tgz", + "integrity": "sha512-uMHhvmDmwFiLuiozaoRYTZHqMFuu3fem/X9JY78lohAuLoh8LSGGC8JqUzo2cVuBsBxkjMZNk56CE7ock8Xxkw==", "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^8.0.0", + "@babel/helper-validator-identifier": "^8.0.6", "js-tokens": "^10.0.0" }, "engines": { @@ -45,19 +45,19 @@ } }, "node_modules/@babel/core": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-8.0.5.tgz", - "integrity": "sha512-2/oWkgTbBYoqioCWAE4XJobOrzwxTDa5/XjDP3tJ1BhDr/owcd9qnXBp4xc3/2G5X4bvXM04nrxbRJxFcXAxFQ==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-8.0.6.tgz", + "integrity": "sha512-5zwYt1V4ji3mXKKRIAkohSJ29WepxNwxQi+7y7P5AcVPh1dcxnZBU2cfxHnPwS+u9+lZnXPsv30TQaF2xg+Tfg==", "license": "MIT", "dependencies": { - "@babel/code-frame": "^8.0.0", - "@babel/generator": "^8.0.5", - "@babel/helper-compilation-targets": "^8.0.5", + "@babel/code-frame": "^8.0.6", + "@babel/generator": "^8.0.6", + "@babel/helper-compilation-targets": "^8.0.6", "@babel/helpers": "^8.0.5", - "@babel/parser": "^8.0.5", + "@babel/parser": "^8.0.6", "@babel/template": "^8.0.0", - "@babel/traverse": "^8.0.5", - "@babel/types": "^8.0.5", + "@babel/traverse": "^8.0.6", + "@babel/types": "^8.0.6", "@types/gensync": "^1.0.5", "convert-source-map": "^2.0.0", "empathic": "^2.0.1", @@ -76,9 +76,9 @@ } }, "node_modules/@babel/eslint-parser": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/@babel/eslint-parser/-/eslint-parser-8.0.5.tgz", - "integrity": "sha512-q4c+FpoJ/FKeiLnZ9Os3vWnSlZw3dBmlTUx6vuITRqbd8yn4hgUB0/OOJ+gi5lt+mOx/0IW2rXINR98JJllfag==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/eslint-parser/-/eslint-parser-8.0.6.tgz", + "integrity": "sha512-mXx93HLovamLnDDLQ4i48gm4HHtaoqJrbMAuJqrMfpCGJkJ5M4sXhRgKGTrxykg1P8tAN5myrJxsRZuMuTq+OQ==", "license": "MIT", "dependencies": { "eslint-scope": "^9.1.0", @@ -94,13 +94,13 @@ } }, "node_modules/@babel/generator": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-8.0.5.tgz", - "integrity": "sha512-f/TuhuMAxJqhwxEGNsJrswuG9VHmh0oNFoQoo6TbpgtFAz9wYZXcTAcWZMHfp7ljesr0RG04bp3Aos9GI59L7w==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-8.0.6.tgz", + "integrity": "sha512-/pcanjdRsonCexCuxgGD8fKnBLLzk2i9uM4kmcmGyyev2R5YDs+rafMYZ89RvaGY1mW8T7C5au0TyYaEeHBGng==", "license": "MIT", "dependencies": { - "@babel/parser": "^8.0.5", - "@babel/types": "^8.0.5", + "@babel/parser": "^8.0.6", + "@babel/types": "^8.0.6", "@jridgewell/gen-mapping": "0.4.0-beta.0", "@jridgewell/trace-mapping": "^0.3.31", "@types/jsesc": "^2.5.0", @@ -111,15 +111,15 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-8.0.5.tgz", - "integrity": "sha512-Qk8ahMGooH5mz6uuhoDvfZGkUf/Mf3RTBucVVl4MKx4LKMTv872TeW8O92h15iVtlN8wAROBIpI1aV6x1z0LCQ==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-8.0.6.tgz", + "integrity": "sha512-HTrtZ4+iA0+kzdbCgvdSk1U4pEl3BvpGnlUbualCGNbrVhhQvhgBzurTUmqpWyvyHbWYWaIfdx2mOlL9LJjsew==", "license": "MIT", "dependencies": { "@babel/compat-data": "^8.0.5", "@babel/helper-validator-option": "^8.0.0", "browserslist": "^4.24.0", - "lru-cache": "^11.0.0", + "flru": "1.0.2", "verkit": "^0.3.2" }, "engines": { @@ -127,9 +127,9 @@ } }, "node_modules/@babel/helper-globals": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-8.0.0.tgz", - "integrity": "sha512-lLozHOM6sWWlxNo8CYqHy4MBZeTvHXNgVPBfPOGsjPKUzHC2Az9QwB6gxdQmpwHl6GlQtbGgS+lj5887guDiLw==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-8.0.6.tgz", + "integrity": "sha512-OdnA/N52LhzFpeUuVz5YGuSWjc7TPHiFmVYZbPlSgExocO2GwbxVpSni9MVs5wwW+wZ4kPuwBEBN9pIPDYz1mA==", "license": "MIT", "engines": { "node": "^22.18.0 || >=24.11.0" @@ -148,18 +148,18 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.6.tgz", + "integrity": "sha512-+6HYEBnsDRZHJtUTAEfMQd0zMDqdFHS7C+5vAsEZqBUr3vEMWOWlHJ0Jz1gS8jJIlCvLsuA6ivmeoqzH936YcA==", "license": "MIT", "engines": { "node": "^22.18.0 || >=24.11.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.6.tgz", + "integrity": "sha512-HYx0YmuchRxRSpnjvcNffnPS8DCauKNmnumIuHG3jnQAVewRA7kfJGwF13ayEpSZs7s4R/oBH4eppuXcAQd5JQ==", "license": "MIT", "engines": { "node": "^22.18.0 || >=24.11.0" @@ -188,12 +188,12 @@ } }, "node_modules/@babel/parser": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.5.tgz", - "integrity": "sha512-51RXvQNFakaS0bTpYiGkxNbUVwkPO4kONv6EVLorZABxsx+KZ6Z7uSYvi/wmKS/+X+rfj9RvOw0/ZNh+cmI0Rw==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.6.tgz", + "integrity": "sha512-LpGDIYJAzc3Y3PT9x+FxBrGYu2PlWxLmyN7SwPTzaX0LSwEmFJBsRnPBqmaXF2r1v+Zhz6lNiEJ+7yKtM5+Ugg==", "license": "MIT", "dependencies": { - "@babel/types": "^8.0.5" + "@babel/types": "^8.0.6" }, "bin": { "parser": "bin/babel-parser.js" @@ -247,17 +247,17 @@ } }, "node_modules/@babel/traverse": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-8.0.5.tgz", - "integrity": "sha512-XFfnuvapSc/vJOcUO7kwORSvpBIvraofKEZ2dhT0PjiF21BRCD7YbAFC8UEeDJNeLoQz82/gVqzgX5hCzkCbdg==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-8.0.6.tgz", + "integrity": "sha512-ilYV+qLStGzSKLipJlETdvdtpK3rAuwfCmCoGKv15MU4EoeIuLkespSNoNLFGOPSciEHx9TLWtN5mv0fz5w5mQ==", "license": "MIT", "dependencies": { - "@babel/code-frame": "^8.0.0", - "@babel/generator": "^8.0.5", - "@babel/helper-globals": "^8.0.0", - "@babel/parser": "^8.0.5", + "@babel/code-frame": "^8.0.6", + "@babel/generator": "^8.0.6", + "@babel/helper-globals": "^8.0.6", + "@babel/parser": "^8.0.6", "@babel/template": "^8.0.0", - "@babel/types": "^8.0.5", + "@babel/types": "^8.0.6", "obug": "^2.1.1" }, "engines": { @@ -265,13 +265,13 @@ } }, "node_modules/@babel/types": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.5.tgz", - "integrity": "sha512-eVdMqi3ej5aHhyQ2Si6yD2cAWeV8FJK9UrhK5aL0Sd8hu5GhT+YswhVNbVheOGVYMg8kuGuMaUpkB3stjj4z8A==", + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.6.tgz", + "integrity": "sha512-c+xgSWboV2pdwXP67BUWDVRHi2BO5Q3KwxyFVs7taVTY+fsK47/L51ZXQoV9rozHJDNHvJ4v4WEsI/IlxV2l2A==", "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" + "@babel/helper-string-parser": "^8.0.6", + "@babel/helper-validator-identifier": "^8.0.6" }, "engines": { "node": "^22.18.0 || >=24.11.0" @@ -300,16 +300,16 @@ } }, "node_modules/@es-joy/jsdoccomment": { - "version": "0.97.0", - "resolved": "https://registry.npmjs.org/@es-joy/jsdoccomment/-/jsdoccomment-0.97.0.tgz", - "integrity": "sha512-EP8uoFfh6+GsdGCduYtmWAW0h7AO+Ayik9Vh5YbA2r/3N6lmJKkCNZX+q3QBXC1K6ixjQ/9igF2b7WVvLm063g==", + "version": "0.98.0", + "resolved": "https://registry.npmjs.org/@es-joy/jsdoccomment/-/jsdoccomment-0.98.0.tgz", + "integrity": "sha512-QS+b7I2aaSM+gZfDz1b9E4B367JJ2qTjFXRKF289iKkh9sYdZ21Q5MiF/Duzouy3J0C4VhM3I/8f2F4+zT8gYQ==", "license": "MIT", "dependencies": { "@types/estree": "^1.0.9", - "@typescript-eslint/types": "^8.69.0", - "comment-parser": "1.4.8", + "@typescript-eslint/types": "^8.70.0", + "comment-parser": "1.4.9", "esquery": "^1.7.0", - "jsdoc-type-pratt-parser": "~9.2.1" + "jsdoc-type-pratt-parser": "~9.2.2" }, "engines": { "node": "^22.22.2 || >=24.15.0" @@ -910,9 +910,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.11.24", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.24.tgz", - "integrity": "sha512-hYrgxie335U08WqICoGqKRzV1HFXv6zdxwJE4ekCb80CM9a0SVVsN4QPwT67RraRo+9h8IATk6uxHJw7QSkdOg==", + "version": "2.11.26", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.26.tgz", + "integrity": "sha512-GLQdD3y6UF8iVuMJl5fHgE4jdn/ua7n+toKfLgNlg3BqQtOZjpy68T8Tup8/wGWZCDlm7KMg7tPb4MPn7oN0TQ==", "license": "Apache-2.0", "bin": { "baseline-browser-mapping": "dist/cli.cjs" @@ -934,9 +934,9 @@ } }, "node_modules/browserslist": { - "version": "4.29.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.0.tgz", - "integrity": "sha512-3GSvyjvDI4Dur1Meg2BekJquu5uF+9R9a1+5M1Mde192eZoXbeXjzgOsgqPS2V8D5wrrip0gR5Hf/GhWQ9ZzaA==", + "version": "4.29.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", + "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", "funding": [ { "type": "opencollective", @@ -953,10 +953,10 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.11.23", - "caniuse-lite": "^1.0.30001810", - "electron-to-chromium": "^1.5.427", - "node-releases": "^2.0.55", + "baseline-browser-mapping": "^2.11.26", + "caniuse-lite": "^1.0.30001813", + "electron-to-chromium": "^1.5.439", + "node-releases": "^2.0.57", "update-browserslist-db": "^1.3.3" }, "bin": { @@ -980,9 +980,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001810", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", - "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", + "version": "1.0.30001814", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001814.tgz", + "integrity": "sha512-/Uaf1lAzr59XcMpW0o96WoEfr+VXK2OX4U9AgFoiSHsVJ4HppnIFUjtYzsyDH2+tgANaQb2/oxYGwCPapN1FpA==", "funding": [ { "type": "opencollective", @@ -1029,9 +1029,9 @@ } }, "node_modules/comment-parser": { - "version": "1.4.8", - "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-1.4.8.tgz", - "integrity": "sha512-rKZTGo4fzKYna8UcL0isTg5wkBNla7bxTypLwZQXjIdi++IdP1OJ41rI5Mti3/jltkPujbu4i9LIARYA+zpotQ==", + "version": "1.4.9", + "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-1.4.9.tgz", + "integrity": "sha512-+2AvZKjJaNq9GQljm3tr0utwrig5BL+jgJGvz5rDpGKNIp+ojcRXWUwBbh/sGIi1QCprR6PsKFakub+w1v+4hQ==", "license": "MIT", "engines": { "node": ">= 12.0.0" @@ -1116,9 +1116,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.429", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.429.tgz", - "integrity": "sha512-/1ENIE3cx4HTIx4IfPZFaOunJmsrSVTnj6coXoRVbiJUbkeTyFkJvBeWGkdgh08OhFbxYLMT1kbkwFVSarq6Ow==", + "version": "1.5.443", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.443.tgz", + "integrity": "sha512-TDJG36L9A3CWWwZ97HKaE+Iz1sW80pNp6sAU0owzCIV3Zc9eVyRqZK8Vz6NKuMIU8WjP0pIkoqCa0VRf/yFD0Q==", "license": "ISC" }, "node_modules/empathic": { @@ -1152,9 +1152,9 @@ } }, "node_modules/eslint": { - "version": "10.10.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.10.0.tgz", - "integrity": "sha512-NPXn6r5zl4uET1DAVPaOwzX3rut4c0wcmw3dWJAfOsTM5+TogXo0DDjz8pwm/hL8cyVNpHqeK4JpN0NjnyFFNw==", + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.11.0.tgz", + "integrity": "sha512-P7a6UEEqb9G95MYAtqkmsTbVXIYyzIfl6NGOIJk162PaahFxFyeGcrlXYFSiagECg4sEm8IseJdZBKR3rx6MsQ==", "license": "MIT", "workspaces": [ "packages/*" @@ -1222,16 +1222,16 @@ } }, "node_modules/eslint-plugin-jsdoc": { - "version": "64.3.10", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-64.3.10.tgz", - "integrity": "sha512-oevWxylnQJm/f4S5WfvqG/pdmX9Ndsjpfm51QpTFfG8MF92999sp4hm4/GxyboaAQRFngLRDL4iKHZz5hPs4Xw==", + "version": "64.5.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-64.5.4.tgz", + "integrity": "sha512-xfRhXPSSWT612muJ6XTvxuVJ8zYHv99qNgBqPVqF28aSqoy4s4XbSkrWXnYA2hl5ejC02yEpk6fqkj3BmJ6Xbg==", "license": "BSD-3-Clause", "dependencies": { - "@es-joy/jsdoccomment": "~0.97.0", + "@es-joy/jsdoccomment": "~0.98.0", "@es-joy/resolve.exports": "1.2.0", - "@typescript-eslint/utils": "^8.69.0", + "@typescript-eslint/utils": "^8.70.0", "are-docs-informative": "^0.1.1", - "comment-parser": "1.4.8", + "comment-parser": "1.4.9", "debug": "^4.4.3", "escape-string-regexp": "^5.0.0", "espree": "^11.2.0", @@ -1247,7 +1247,13 @@ "node": "^22.22.2 || >=24.15.0" }, "peerDependencies": { - "eslint": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0" + "eslint": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0", + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } } }, "node_modules/eslint-plugin-jsdoc/node_modules/escape-string-regexp": { @@ -1280,9 +1286,9 @@ } }, "node_modules/eslint-plugin-regexp": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-regexp/-/eslint-plugin-regexp-3.3.0.tgz", - "integrity": "sha512-TT0JTQbW7CRKohntpGAsMdQ1K3MsmJK4gdAhKJtIwlJ0JvuYxauymFgJdvqxIY4lqNj0EuRZWVrymQtEdJYEFg==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-regexp/-/eslint-plugin-regexp-3.3.1.tgz", + "integrity": "sha512-58bGUci/AaWAo1ETj6Lsnmy6QXwnSyVONRA805828nnqp7vrTwdaQMc02vtowmYB3A5OB3SLr3r5HgmfxX4dLg==", "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", @@ -1508,6 +1514,15 @@ "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", "license": "ISC" }, + "node_modules/flru": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/flru/-/flru-1.0.2.tgz", + "integrity": "sha512-kWyh8ADvHBFz6ua5xYOPnUroZTT/bwWfrCeL0Wj1dzG4/YOmOcfJ99W8dOVyyynJN35rZ9aCOtHChqQovV7yog==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/format": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", @@ -1673,9 +1688,9 @@ } }, "node_modules/jsdoc-type-pratt-parser": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-9.2.1.tgz", - "integrity": "sha512-V4Ww4EHnTcTLSOMoB0FsF72JhQvcAsriCm/LWnxJeGWoxIjEL2l9na11abQok5SYShq8m0Gl02el/xAbTCulvQ==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-9.2.2.tgz", + "integrity": "sha512-eFY7Rlscvv/BpY6UsVsjKwMhu1tJO/F4U/vC9g3yeRy/JqszuO1UP6Vdq+85G+3FP6fEw/sWR8a5ftPFY86csw==", "license": "MIT", "dependencies": { "@types/estree": "^1.0.9", @@ -1784,15 +1799,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/lru-cache": { - "version": "11.5.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", - "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/markdown-table": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", @@ -2679,9 +2685,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.55", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.55.tgz", - "integrity": "sha512-mIrE/Cw9y+9Au6dS5vDKDhQza9YvG6w+ZrS6X+ZzA7yFW/soAeaups4Qzn1bL6g5FVy8WtP79+0j82oPIbqRjQ==", + "version": "2.0.57", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.57.tgz", + "integrity": "sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==", "license": "MIT", "engines": { "node": ">=18" diff --git a/tools/eslint/package.json b/tools/eslint/package.json index 8357472e24ca..70fb0a1825fe 100644 --- a/tools/eslint/package.json +++ b/tools/eslint/package.json @@ -3,17 +3,17 @@ "version": "0.0.0", "private": true, "dependencies": { - "@babel/core": "^8.0.5", - "@babel/eslint-parser": "^8.0.5", + "@babel/core": "^8.0.6", + "@babel/eslint-parser": "^8.0.6", "@babel/plugin-syntax-import-defer": "^8.0.1", "@babel/plugin-syntax-import-source": "^8.0.1", "@eslint/js": "^10.0.1", "@eslint/markdown": "^8.0.3", "@stylistic/eslint-plugin": "^5.10.0", - "eslint": "^10.10.0", + "eslint": "^10.11.0", "eslint-formatter-tap": "^9.0.1", - "eslint-plugin-jsdoc": "^64.3.10", - "eslint-plugin-regexp": "^3.3.0", + "eslint-plugin-jsdoc": "^64.5.4", + "eslint-plugin-regexp": "^3.3.1", "globals": "^17.12.0" } } From 9618da6fe35571a0e929070ee986517473efb68c Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Sat, 3 Oct 2026 23:43:02 +0200 Subject: [PATCH 08/98] test: remove duplicated cases in permission fs-read fixture The fs.createReadStream(blockedFile) and fs.lstatSync(bufferBlockedFile) assertions were each present twice in the same block with identical arguments and expectations, so the second copy added no coverage. Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66450 Reviewed-By: Luigi Pinca Reviewed-By: Filip Skokan --- test/fixtures/permission/fs-read.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/fixtures/permission/fs-read.js b/test/fixtures/permission/fs-read.js index 1ad12707ba55..511c9bf027b1 100644 --- a/test/fixtures/permission/fs-read.js +++ b/test/fixtures/permission/fs-read.js @@ -76,17 +76,6 @@ const regularFile = __filename; permission: 'FileSystemRead', resource: path.toNamespacedPath(blockedFile), })).then(common.mustCall()); - - assert.rejects(() => { - return new Promise((_resolve, reject) => { - const stream = fs.createReadStream(blockedFile); - stream.on('error', reject); - }); - }, common.expectsError({ - code: 'ERR_ACCESS_DENIED', - permission: 'FileSystemRead', - resource: path.toNamespacedPath(blockedFile), - })).then(common.mustCall()); } // fs.stat @@ -711,11 +700,6 @@ const regularFile = __filename; }, common.expectsError({ code: 'ERR_ACCESS_DENIED', })); - assert.throws(() => { - fs.lstatSync(bufferBlockedFile); - }, common.expectsError({ - code: 'ERR_ACCESS_DENIED', - })); // doesNotThrow fs.lstat(regularFile, (err) => { From b2773d5e4cf217fb4653a2b92586834743e8b9d2 Mon Sep 17 00:00:00 2001 From: Daniel Pinto <1619617+Cayan@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:37:42 -0300 Subject: [PATCH 09/98] doc: add missing return types in cluster.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `worker.isConnected()` and `worker.isDead()` described their boolean result only in prose, so the documentation tooling had no return type for them. Signed-off-by: Daniel Pinto <1619617+Cayan@users.noreply.github.com> Assisted-by: a frontier reasoning model PR-URL: https://github.com/nodejs/node/pull/66453 Reviewed-By: René Reviewed-By: Luigi Pinca --- doc/api/cluster.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/api/cluster.md b/doc/api/cluster.md index 936a535977c5..6d3f9e6fc659 100644 --- a/doc/api/cluster.md +++ b/doc/api/cluster.md @@ -489,6 +489,8 @@ While a worker is alive, this is the key that indexes it in added: v0.11.14 --> +* Returns: {boolean} + This function returns `true` if the worker is connected to its primary via its IPC channel, `false` otherwise. A worker is connected to its primary after it has been created. It is disconnected after the `'disconnect'` event is emitted. @@ -499,6 +501,8 @@ has been created. It is disconnected after the `'disconnect'` event is emitted. added: v0.11.14 --> +* Returns: {boolean} + This function returns `true` if the worker's process has terminated (either because of exiting or being signaled). Otherwise, it returns `false`. From c1073f31ded6b902bf102641f4b3220cff56091c Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:56:13 -0700 Subject: [PATCH 10/98] ffi: fix exportArrayBuffer type check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use isArrayBuffer() instead of comparing Object.prototype.toString() output. The old check depended on Symbol.toStringTag, so a real ArrayBuffer with a custom tag was rejected, and a plain object tagged 'ArrayBuffer' passed and failed later in native code with a misleading error. Signed-off-by: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Assisted-by: claude:opus-5.5 PR-URL: https://github.com/nodejs/node/pull/66455 Fixes: https://github.com/nodejs/node/issues/66454 Reviewed-By: René Reviewed-By: Paolo Insogna Reviewed-By: Anna Henningsen --- lib/ffi.js | 4 ++-- test/ffi/test-ffi-memory.js | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/ffi.js b/lib/ffi.js index 2a9db50fa51c..82663a9710dd 100644 --- a/lib/ffi.js +++ b/lib/ffi.js @@ -8,7 +8,6 @@ const { ObjectFreeze, ObjectGetOwnPropertyDescriptor, ObjectKeys, - ObjectPrototypeToString, ReflectConstruct, SafeWeakMap, SafeWeakRef, @@ -18,6 +17,7 @@ const { const { Buffer } = require('buffer'); const { emitExperimentalWarning } = require('internal/util'); const { + isArrayBuffer, isDataView, isArrayBufferView, isSharedArrayBuffer, @@ -311,7 +311,7 @@ function exportBuffer(source, data, len) { function exportArrayBuffer(source, data, len) { checkFFIPermission(); - if (ObjectPrototypeToString(source) !== '[object ArrayBuffer]') { + if (!isArrayBuffer(source)) { throw new ERR_INVALID_ARG_TYPE('arrayBuffer', 'ArrayBuffer', source); } diff --git a/test/ffi/test-ffi-memory.js b/test/ffi/test-ffi-memory.js index 1ca54ce897cb..3bcda27e1765 100644 --- a/test/ffi/test-ffi-memory.js +++ b/test/ffi/test-ffi-memory.js @@ -238,6 +238,11 @@ test('ffi exportString and exportBuffer copy data into native memory', () => { ffi.exportArrayBuffer(arrayBuffer, arrayBufferPtr, 4); assert.deepStrictEqual([...ffi.toBuffer(arrayBufferPtr, 4)], [8, 9, 10, 11]); + const taggedArrayBuffer = new Uint8Array([12, 13, 14, 15]).buffer; + Object.defineProperty(taggedArrayBuffer, Symbol.toStringTag, { value: 'Custom' }); + ffi.exportArrayBuffer(taggedArrayBuffer, arrayBufferPtr, 4); + assert.deepStrictEqual([...ffi.toBuffer(arrayBufferPtr, 4)], [12, 13, 14, 15]); + const viewPtr = alloc(8); const viewSource = new Uint16Array([0x0102, 0x0304, 0x0506]); const middleBytes = new Uint8Array(viewSource.buffer, 2, 2); @@ -318,6 +323,10 @@ test('ffi validates memory access arguments', () => { assert.throws(() => ffi.exportBuffer(Buffer.from([1]), ptr, -1), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => ffi.exportBuffer(Buffer.from([1, 2]), ptr, 1), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => ffi.exportArrayBuffer('bad', ptr, 4), { code: 'ERR_INVALID_ARG_TYPE' }); + assert.throws(() => ffi.exportArrayBuffer({ [Symbol.toStringTag]: 'ArrayBuffer', byteLength: 1 }, ptr, 4), { + code: 'ERR_INVALID_ARG_TYPE', + message: /The "arrayBuffer" argument must be an instance of ArrayBuffer/, + }); assert.throws(() => ffi.exportArrayBuffer(new ArrayBuffer(1), ptr, -1), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => ffi.exportArrayBuffer(new ArrayBuffer(2), ptr, 1), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => ffi.exportArrayBufferView('bad', ptr, 4), { code: 'ERR_INVALID_ARG_TYPE' }); From 20441b8fc04325176dae6d0b8a8c497b84afcf9d Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 4 Oct 2026 09:30:28 +0200 Subject: [PATCH 11/98] test: run node-bench tools test sequentially The test and its benchmark subprocesses allocate large histograms. Run it sequentially to reduce memory pressure on constrained CI hosts. Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/66338 Reviewed-By: Chemi Atlow --- .../test-benchmark-node-bench-tools.js | 3 +++ 1 file changed, 3 insertions(+) rename test/{parallel => sequential}/test-benchmark-node-bench-tools.js (98%) diff --git a/test/parallel/test-benchmark-node-bench-tools.js b/test/sequential/test-benchmark-node-bench-tools.js similarity index 98% rename from test/parallel/test-benchmark-node-bench-tools.js rename to test/sequential/test-benchmark-node-bench-tools.js index 3d248b45c0e3..84ec4223b52c 100644 --- a/test/parallel/test-benchmark-node-bench-tools.js +++ b/test/sequential/test-benchmark-node-bench-tools.js @@ -1,6 +1,9 @@ // Flags: --no-warnings 'use strict'; +// This test and its subprocesses allocate large, high-precision histograms. +// Run sequentially to avoid exhausting memory on constrained CI hosts. + require('../common'); const assert = require('assert'); const { spawnSync } = require('child_process'); From c6f0614785eded143f7a95a3cb4fd184ba074476 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sun, 4 Oct 2026 21:16:34 +0200 Subject: [PATCH 12/98] crypto: remove select Web Crypto warnings Remove ExperimentalWarning from algorithms and methods that are on their way to an unflagged browser implementation. Signed-off-by: Filip Skokan PR-URL: https://github.com/nodejs/node/pull/66312 Reviewed-By: James M Snell --- lib/internal/crypto/util.js | 23 ++++++++++--------- lib/internal/crypto/webcrypto.js | 7 ------ .../test-webcrypto-prototype-pollution.mjs | 4 ++-- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/internal/crypto/util.js b/lib/internal/crypto/util.js index 88aa42b08522..3633bce9cb18 100644 --- a/lib/internal/crypto/util.js +++ b/lib/internal/crypto/util.js @@ -567,26 +567,19 @@ function getConditionalAlgorithms() { }; } -// Experimental algorithms -const experimentalAlgorithms = [ +// Experimental algorithms that emit warnings. +const experimentalAlgorithmsWithWarnings = [ 'AES-OCB', 'Argon2d', 'Argon2i', 'Argon2id', - 'ChaCha20-Poly1305', 'cSHAKE128', 'cSHAKE256', 'Ed448', 'KMAC128', 'KMAC256', - 'ML-DSA-44', - 'ML-DSA-65', - 'ML-DSA-87', 'ML-KEM-512', - 'ML-KEM-768', - 'ML-KEM-1024', 'MLKEM768-P256', - 'MLKEM768-X25519', 'MLKEM1024-P384', 'SHA3-256', 'SHA3-384', @@ -596,6 +589,14 @@ const experimentalAlgorithms = [ 'KT128', 'KT256', 'X448', + // Ships unflagged in Chromium >= 155 + // 'ChaCha20-Poly1305' + // 'ML-DSA-44' + // 'ML-DSA-65' + // 'ML-DSA-87' + // 'ML-KEM-768' + // 'ML-KEM-1024' + // 'MLKEM768-X25519' ]; // Transform the algorithm definitions into the operation-keyed structure @@ -624,8 +625,8 @@ function createSupportedAlgorithms(algorithmDefs) { nameMap[operation] ||= new SafeMap(); nameMap[operation].set(StringPrototypeToUpperCase(algorithmName), algorithmName); - // Add experimental warnings for experimental algorithms - if (ArrayPrototypeIncludes(experimentalAlgorithms, algorithmName)) { + // Add warnings for select experimental algorithms + if (ArrayPrototypeIncludes(experimentalAlgorithmsWithWarnings, algorithmName)) { ObjectDefineProperty(result[operation], algorithmName, { get() { emitExperimentalWarning(`The ${algorithmName} Web Crypto API algorithm`); diff --git a/lib/internal/crypto/webcrypto.js b/lib/internal/crypto/webcrypto.js index 02318f93dc99..7ca945cb4f21 100644 --- a/lib/internal/crypto/webcrypto.js +++ b/lib/internal/crypto/webcrypto.js @@ -74,7 +74,6 @@ const { } = require('internal/crypto/util'); const { - emitExperimentalWarning, kEnumerableProperty, lazyDOMException, setOwnProperty, @@ -1473,7 +1472,6 @@ function getPublicKey(key, keyUsages) { } function getPublicKeyImpl(key, keyUsages) { - emitExperimentalWarning('The getPublicKey Web Crypto API method'); const prefix = prepareSubtleMethod(this, 'getPublicKey', arguments.length, 2); let i = 0; key = convertSubtleArgument(prefix, 'CryptoKey', key, i++); @@ -1501,7 +1499,6 @@ function encapsulateBits(encapsulationAlgorithm, encapsulationKey) { } function encapsulateBitsImpl(encapsulationAlgorithm, encapsulationKey) { - emitExperimentalWarning('The encapsulateBits Web Crypto API method'); const prefix = prepareSubtleMethod( this, 'encapsulateBits', arguments.length, 2); let i = 0; @@ -1563,7 +1560,6 @@ function encapsulateKeyImpl( sharedKeyAlgorithm, extractable, keyUsages) { - emitExperimentalWarning('The encapsulateKey Web Crypto API method'); const prefix = prepareSubtleMethod( this, 'encapsulateKey', arguments.length, 5); let i = 0; @@ -1642,7 +1638,6 @@ function decapsulateBits(decapsulationAlgorithm, decapsulationKey, ciphertext) { } function decapsulateBitsImpl(decapsulationAlgorithm, decapsulationKey, ciphertext) { - emitExperimentalWarning('The decapsulateBits Web Crypto API method'); const prefix = prepareSubtleMethod( this, 'decapsulateBits', arguments.length, 3); let i = 0; @@ -1707,7 +1702,6 @@ function decapsulateKeyImpl( sharedKeyAlgorithm, extractable, keyUsages) { - emitExperimentalWarning('The decapsulateKey Web Crypto API method'); const prefix = prepareSubtleMethod( this, 'decapsulateKey', arguments.length, 6); let i = 0; @@ -1785,7 +1779,6 @@ class SubtleCrypto { // Implements https://wicg.github.io/webcrypto-modern-algos/#SubtleCrypto-method-supports static supports(operation, algorithm, lengthOrAdditionalAlgorithm = null) { - emitExperimentalWarning('The supports Web Crypto API method'); webidl ??= require('internal/crypto/webidl'); const prefix = "Failed to execute 'supports' on 'SubtleCrypto'"; webidl.requiredArguments(arguments.length, 2, { prefix }); diff --git a/test/parallel/test-webcrypto-prototype-pollution.mjs b/test/parallel/test-webcrypto-prototype-pollution.mjs index b6f2c07f81a4..45b5dfc8389d 100644 --- a/test/parallel/test-webcrypto-prototype-pollution.mjs +++ b/test/parallel/test-webcrypto-prototype-pollution.mjs @@ -20,8 +20,8 @@ const TypedArrayPrototype = Object.getPrototypeOf(Uint8Array.prototype); const data = new TextEncoder().encode('prototype pollution'); const modulusLength = getFips() === 1 ? 2048 : 1024; -// Avoids SubtleCrypto.supports(), which warns and invokes the registry's -// experimental-algorithm getters. +// Avoids SubtleCrypto.supports(), which can invoke the registry's +// experimental-algorithm warning getters. function supports(operation, name) { return Object.hasOwn(kSupportedAlgorithms[operation] ?? {}, name); } From 4ba6ce1b0c714f74a18976f385192dbb338d60b3 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 5 Oct 2026 07:16:57 +0200 Subject: [PATCH 13/98] stream: port remaining SonicBoom tests for Utf8Stream Port the SonicBoom tests that could not be imported directly because they depend on the third-party proxyquire module, along with the maxWriteRetries option they exercise, into the Utf8Stream module. Adds maxWriteRetries support, which bounds consecutive EAGAIN/EBUSY retry attempts and resets the counter on forward progress, plus drop-event and flush-callback edge-case coverage. Fixes: https://github.com/nodejs/node/issues/58955 Signed-off-by: Matteo Collina Assisted-by: pi PR-URL: https://github.com/nodejs/node/pull/66275 Reviewed-By: Paolo Insogna Reviewed-By: James M Snell --- lib/internal/streams/fast-utf8-stream.js | 34 +++++- test/parallel/test-fastutf8stream-flush.js | 108 +++++++++++++++++++ test/parallel/test-fastutf8stream-retry.js | 114 +++++++++++++++++++++ test/parallel/test-fastutf8stream-write.js | 64 ++++++++++++ 4 files changed, 317 insertions(+), 3 deletions(-) diff --git a/lib/internal/streams/fast-utf8-stream.js b/lib/internal/streams/fast-utf8-stream.js index a11fbf3bbc0e..d97f1cae3257 100644 --- a/lib/internal/streams/fast-utf8-stream.js +++ b/lib/internal/streams/fast-utf8-stream.js @@ -87,6 +87,8 @@ class Utf8Stream extends EventEmitter { #mode = 0o666; #retryEAGAIN = () => true; #mkdir = false; + #maxWriteRetries = 0; + #writeRetries = 0; #writingBuf = ''; #write; #flush; @@ -132,6 +134,7 @@ class Utf8Stream extends EventEmitter { append = true, mkdir, retryEAGAIN, + maxWriteRetries, fsync, contentMode = kContentModeUtf8, mode, @@ -165,12 +168,14 @@ class Utf8Stream extends EventEmitter { this.#mode = mode; this.#retryEAGAIN = retryEAGAIN || (() => true); this.#mkdir = mkdir || false; + this.#maxWriteRetries = maxWriteRetries || 0; validateUint32(this.#hwm, 'options.hwm'); validateUint32(this.#minLength, 'options.minLength'); validateUint32(this.#maxLength, 'options.maxLength'); validateUint32(this.#maxWrite, 'options.maxWrite'); validateUint32(this.#periodicFlush, 'options.periodicFlush'); + validateUint32(this.#maxWriteRetries, 'options.maxWriteRetries'); validateBoolean(this.#sync, 'options.sync'); validateBoolean(this.#fsync, 'options.fsync'); validateBoolean(this.#append, 'options.append'); @@ -377,7 +382,13 @@ class Utf8Stream extends EventEmitter { #release(err, n) { if (err) { - if ((err.code === 'EAGAIN' || err.code === 'EBUSY') && + const isRetryableErr = (err.code === 'EAGAIN' || err.code === 'EBUSY'); + if (isRetryableErr) { + this.#writeRetries++; + } + const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries; + + if (isRetryableErr && !retriesExhausted && this.#retryEAGAIN(err, this.#writingBuf.length, this.#len - this.#writingBuf.length)) { if (this.#sync) { // This error code should not happen in sync mode, because it is @@ -402,6 +413,13 @@ class Utf8Stream extends EventEmitter { return; } + // Reset the retry counter only once real forward progress (n > 0) is + // confirmed. Otherwise a stuck destination would never accumulate past + // one retry and `maxWriteRetries` could never trigger. + if (n > 0) { + this.#writeRetries = 0; + } + this.emit('write', n); const releasedBufObj = releaseWritingBuf(this.#writingBuf, this.#len, n); this.#len = releasedBufObj.len; @@ -639,6 +657,7 @@ class Utf8Stream extends EventEmitter { } try { const n = this.#fs.writeSync(this.#fd, buf); + this.#writeRetries = 0; buf = buf.subarray(n); this.#len = MathMax(this.#len - n, 0); if (buf.length <= 0) { @@ -647,7 +666,11 @@ class Utf8Stream extends EventEmitter { } } catch (err) { const shouldRetry = err.code === 'EAGAIN' || err.code === 'EBUSY'; - if (shouldRetry && !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) { + if (shouldRetry) { + this.#writeRetries++; + } + const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries; + if (!shouldRetry || retriesExhausted || !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) { throw err; } @@ -677,6 +700,7 @@ class Utf8Stream extends EventEmitter { } try { const n = this.#fs.writeSync(this.#fd, buf, 'utf8'); + this.#writeRetries = 0; const releasedBufObj = releaseWritingBuf(buf, this.#len, n); buf = releasedBufObj.writingBuf; this.#len = releasedBufObj.len; @@ -685,7 +709,11 @@ class Utf8Stream extends EventEmitter { } } catch (err) { const shouldRetry = err.code === 'EAGAIN' || err.code === 'EBUSY'; - if (shouldRetry && !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) { + if (shouldRetry) { + this.#writeRetries++; + } + const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries; + if (!shouldRetry || retriesExhausted || !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) { throw err; } diff --git a/test/parallel/test-fastutf8stream-flush.js b/test/parallel/test-fastutf8stream-flush.js index 195d5e83c9c7..60238ed76908 100644 --- a/test/parallel/test-fastutf8stream-flush.js +++ b/test/parallel/test-fastutf8stream-flush.js @@ -4,9 +4,12 @@ const common = require('../common'); const tmpdir = require('../common/tmpdir'); const assert = require('node:assert'); const { + open, openSync, readFile, writeFileSync, + write, + writeSync, } = require('node:fs'); const { join } = require('node:path'); const { Utf8Stream } = require('node:fs'); @@ -25,6 +28,31 @@ function getTempFile() { runTests(false); runTests(true); +// Flush cb is invoked when flushing before 'ready' while the stream is +// still opening (async mode only; sync mode writes synchronously). +{ + const dest = getTempFile(); + + const stream = new Utf8Stream({ + dest, + minLength: 4096, + sync: false, + fs: { + open(file, flags, mode, cb) { + process.nextTick(() => { + assert.ok(stream.write('hello world\n')); + stream.flush(common.mustSucceed(() => { + stream.destroy(); + })); + open(file, flags, mode, cb); + }); + }, + }, + }); + + stream.on('ready', common.mustCall()); +} + function runTests(sync) { { const dest = getTempFile(); @@ -136,4 +164,84 @@ function runTests(sync) { stream.destroy(); stream.flush(common.mustCall(assert.ok)); } + + { + // Flush cb is invoked with the error when the underlying write fails. + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + + const err = new Error('other'); + err.code = 'other'; + let first = true; + + const fsOverride = {}; + if (sync) { + fsOverride.writeSync = common.mustCallAtLeast((...args) => { + if (first) { + first = false; + throw err; + } + return writeSync(...args); + }, 1); + } else { + fsOverride.write = common.mustCallAtLeast((...args) => { + const callback = args[args.length - 1]; + if (first) { + first = false; + process.nextTick(callback, err); + return; + } + return write(...args); + }, 1); + } + + const stream = new Utf8Stream({ + fd, + sync, + minLength: 4096, + fs: fsOverride, + }); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write('hello world\n')); + stream.flush(common.mustCall((e) => { + assert.strictEqual(e.code, 'other'); + stream.destroy(); + })); + })); + } + + { + // Flush cb is invoked once the in-flight write completes. + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + + const fsOverride = {}; + if (sync) { + fsOverride.writeSync = common.mustCallAtLeast((...args) => { + stream.flush(common.mustSucceed(() => { + stream.destroy(); + })); + return writeSync(...args); + }, 1); + } else { + fsOverride.write = common.mustCallAtLeast((...args) => { + stream.flush(common.mustSucceed(() => { + stream.destroy(); + })); + return write(...args); + }, 1); + } + + const stream = new Utf8Stream({ + fd, + sync, + minLength: 1, + fs: fsOverride, + }); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write('hello world\n')); + })); + } } diff --git a/test/parallel/test-fastutf8stream-retry.js b/test/parallel/test-fastutf8stream-retry.js index 3381c9beac00..e1b29ed27b3c 100644 --- a/test/parallel/test-fastutf8stream-retry.js +++ b/test/parallel/test-fastutf8stream-retry.js @@ -211,3 +211,117 @@ function runTests(sync) { })); })); } + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + + const err = new Error('EAGAIN'); + err.code = 'EAGAIN'; + let attempts = 0; + + const stream = new Utf8Stream({ + fd, + sync: false, + minLength: 0, + maxWriteRetries: 3, + // retryEAGAIN always returns true ("keep going"), so maxWriteRetries must + // itself cap the number of attempts instead of retrying forever. + retryEAGAIN: () => true, + fs: { + write: common.mustCall((...args) => { + attempts++; + const callback = args[args.length - 1]; + process.nextTick(callback, err); + }, 4), + } + }); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write('hello world\n')); + })); + + stream.once('error', common.mustCall((err) => { + assert.strictEqual(err.code, 'EAGAIN'); + // 1 initial attempt + 3 retries = 4 total fs.write calls, then give up. + assert.strictEqual(attempts, 4); + assert.strictEqual(stream.writing, false); + stream.destroy(); + })); +} + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + + const err = new Error('EAGAIN'); + err.code = 'EAGAIN'; + + const stream = new Utf8Stream({ + fd, + sync: true, + minLength: 0, + maxWriteRetries: 3, + retryEAGAIN: () => true, + fs: { + writeSync: common.mustCall((...args) => { + throw err; + }, 4), + } + }); + + stream.on('ready', common.mustCall(() => { + // Once retries are exhausted, write() must surface the error instead of + // spinning forever on EAGAIN. + assert.throws(() => { + stream.write('hello world\n'); + }, (e) => e.code === 'EAGAIN'); + stream.destroy(); + })); +} + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + + const err = new Error('EAGAIN'); + err.code = 'EAGAIN'; + let call = 0; + + const stream = new Utf8Stream({ + fd, + sync: false, + minLength: 0, + maxWriteRetries: 3, + retryEAGAIN: () => true, + fs: { + write: common.mustCallAtLeast((...args) => { + call++; + const callback = args[args.length - 1]; + if (call % 3 === 0) { + return write(...args); + } + process.nextTick(callback, err); + }, 5), + } + }); + + stream.on('error', common.mustNotCall()); + + stream.on('ready', common.mustCall(() => { + // First burst: calls 1-2 fail with EAGAIN, call 3 succeeds. + assert.ok(stream.write('hello world\n')); + stream.once('drain', common.mustCall(() => { + // Second burst: calls 4-5 fail again. The counter must have been reset + // by the successful write at call 3, so this burst succeeds too. + stream.write('sonic boom\n'); + stream.end(); + })); + })); + + stream.on('finish', common.mustCall(() => { + readFile(dest, 'utf8', common.mustSucceed((data) => { + assert.strictEqual(data, 'hello world\nsonic boom\n'); + })); + })); +} diff --git a/test/parallel/test-fastutf8stream-write.js b/test/parallel/test-fastutf8stream-write.js index a022aead59a9..432b044e165c 100644 --- a/test/parallel/test-fastutf8stream-write.js +++ b/test/parallel/test-fastutf8stream-write.js @@ -268,3 +268,67 @@ function runTests(sync) { assert.strictEqual(stream.maxLength, 65536); stream.end(); } + +// maxLength drop behavior: writing past maxLength must emit 'drop' and +// never write the overflowing chunk (see SonicBoom write tests). +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + + const buf = Buffer.alloc(100).fill('x').toString(); + + const stream = new Utf8Stream({ + fd, + minLength: 101, + maxLength: 102, + sync: false, + fs: { + write: common.mustCall((...args) => { + const data = args[1]; + const callback = args[args.length - 1]; + assert.strictEqual(data.length, buf.length + 2); + process.nextTick(callback, null, data.length); + stream.end(); + }, 1), + } + }); + + stream.on('drop', common.mustNotCall()); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write(buf)); + assert.ok(stream.write('aa')); + })); +} + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + + const buf = Buffer.alloc(100).fill('x').toString(); + + const stream = new Utf8Stream({ + fd, + minLength: 101, + maxLength: 102, + sync: false, + fs: { + write: common.mustCall((...args) => { + const data = args[1]; + const callback = args[args.length - 1]; + assert.strictEqual(data.length, buf.length); + process.nextTick(callback, null, data.length); + }, 1), + } + }); + + stream.on('drop', common.mustCall((data) => { + assert.strictEqual(data.length, 3); + stream.end(); + })); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write(buf)); + assert.ok(stream.write('aaa')); + })); +} From 63c20ee58e20f27cbdad6757634fe6ba36f2f2e1 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 5 Oct 2026 07:17:09 +0200 Subject: [PATCH 14/98] vfs: add ComposableProvider for layered mounts Allow providers to be layered at one mount point. Reads search layers in priority order, while writes copy lower-layer files to the first provider and deletions hide lower copies without changing them. Assisted-by: pi Signed-off-by: Matteo Collina PR-URL: https://github.com/nodejs/node/pull/66235 Reviewed-By: James M Snell Reviewed-By: Paolo Insogna --- doc/api/vfs.md | 51 +++ lib/internal/vfs/providers/composable.js | 416 ++++++++++++++++++ lib/vfs.js | 2 + test/parallel/test-vfs-composable-provider.js | 117 +++++ 4 files changed, 586 insertions(+) create mode 100644 lib/internal/vfs/providers/composable.js create mode 100644 test/parallel/test-vfs-composable-provider.js diff --git a/doc/api/vfs.md b/doc/api/vfs.md index cc2b1ccf3946..3da056de5c06 100644 --- a/doc/api/vfs.md +++ b/doc/api/vfs.md @@ -650,6 +650,56 @@ provider.setReadOnly(); myVfs.writeFileSync('/x.txt', 'fail'); // throws EROFS ``` +## Class: `ComposableProvider` + + + +[`ComposableProvider`][] combines one or more providers in priority order. The first +provider is the writable layer; reads search from first to last. Directories +are merged, with entries in higher-priority layers shadowing entries with the +same name in lower layers. Writes to a lower file copy it to the first provider +before changing it. Removing a file hides lower copies without deleting them. +The first provider must be writable to change the composed file system. + +### `new ComposableProvider(providers)` + + + +* `providers` {VirtualProvider\[]} Non-empty array of providers, ordered from + highest to lowest priority. + +```cjs +const vfs = require('node:vfs'); + +const memory = new vfs.MemoryProvider(); +const disk = new vfs.RealFSProvider('/tmp/vfs-root'); +const combined = vfs.create(new vfs.ComposableProvider([memory, disk])); +combined.writeFileSync('/config.json', '{"debug":true}'); +// The file in memory shadows /tmp/vfs-root/config.json. +``` + +### `composableProvider.providers` + + + +* {VirtualProvider\[]} + +A copy of the ordered provider list. Changes to this array do not affect the +composition. File handles opened before a write continue to refer to the layer +on which they were opened. Watching a path watches only its currently selected +provider, not changes across the entire composition. Symbolic links are +resolved by the provider containing them, not across providers. Traversal +through a symbolic-link directory is not supported by the composition. Renaming +a directory over a directory that exists only in a lower layer is not supported. +Layer selection and copy-up use synchronous provider operations, including +when invoked through the asynchronous VFS API. + ## Class: `RealFSProvider` + +If present, Node.js will look for a configuration file at the specified path. +If the path is not specified, Node.js will look for a `node.config.json` file +in the current working directory. +To specify a custom path, use the `--config-file=path` form. +The space-separated `--config-file path` form is not supported. +Node.js will read the configuration file and apply the settings. The +configuration file should be a JSON file with the following structure. `vX.Y.Z` +in the `$schema` must be replaced with the version of Node.js you are using or +`latest-vX.x` for the latest version of that major release line. + +```json +{ + "$schema": "https://nodejs.org/dist/vX.Y.Z/docs/node-config-schema.json", + "nodeOptions": { + "import": [ + "amaro/strip" + ], + "watch-path": "src", + "watch-preserve-output": true + }, + "test": { + "test-isolation": "process" + }, + "watch": { + "watch-preserve-output": true + } +} +``` + +The configuration file supports namespace-specific options: + +* The `nodeOptions` field contains CLI flags that are allowed in [`NODE_OPTIONS`][]. + +* Namespace fields like `test`, `watch`, and `permission` contain configuration specific to that subsystem. + +The configuration file can target a specific Node.js major version with +`nodeVersion`: + +```json +{ + "nodeVersion": 25, + "nodeOptions": { + "watch-path": "src" + } +} +``` + +To keep multiple version-specific configurations in the same file, use the +`configs` array. Node.js will use the first entry whose `nodeVersion` matches +the current Node.js major version: + +```json +{ + "$schema": "https://nodejs.org/dist/latest-v26.x/docs/node-config-schema.json", + "configs": [ + { + "nodeVersion": 25, + "config": { + "$schema": "https://nodejs.org/dist/latest-v25.x/docs/node-config-schema.json", + "nodeOptions": { + "watch-path": "src" + } + } + } + ] +} +``` + +When `configs` is used, the top level may only contain `$schema` and +`configs`. Each `configs` item must define an integer `nodeVersion` and an +object `config`. A single top-level config does not require `nodeVersion`, but +if present it must match the current Node.js major version. + +When a namespace is present in the +configuration file, Node.js automatically enables the corresponding flag +(e.g., `--test`, `--watch`, `--permission`). This allows you to configure +subsystem-specific options without explicitly passing the flag on the command line. + +For example: + +```json +{ + "test": { + "test-isolation": "process" + } +} +``` + +is equivalent to: + +```bash +node --test --test-isolation=process +``` + +To disable the automatic flag while still using namespace options, you can +explicitly set the flag to `false` within the namespace: + +```json +{ + "test": { + "test": false, + "test-isolation": "process" + } +} +``` + +No-op flags are not supported. +Not all V8 flags are currently supported. + +It is possible to use the [official JSON schema](../node-config-schema.json) +to validate the configuration file, which may vary depending on the Node.js version. +Each key in the configuration file corresponds to a flag that can be passed +as a command-line argument. The value of the key is the value that would be +passed to the flag. + +For example, the configuration file above is equivalent to +the following command-line arguments: + +```bash +node --import amaro/strip --watch-path=src --watch-preserve-output --test-isolation=process +``` + +The priority in configuration is as follows: + +1. NODE\_OPTIONS and command-line options +2. Dotenv NODE\_OPTIONS +3. Configuration file + +Values in the configuration file will not override the values in the environment +variables, command-line options, or the `NODE_OPTIONS` env file parsed by the +`--env-file` flag. + +Keys cannot be duplicated within the same or different namespaces. + +The configuration parser will throw an error if the configuration file contains +unknown keys or keys that cannot be used in a namespace. + +Node.js will not sanitize or perform validation on the user-provided configuration, +so **NEVER** use untrusted configuration files. + ### `--cpu-prof` - -> Stability: 1.2 - Release candidate - -If present, Node.js will look for a configuration file at the specified path. -If the path is not specified, Node.js will look for a `node.config.json` file -in the current working directory. -To specify a custom path, use the `--experimental-config-file=path` form. -The space-separated `--experimental-config-file path` form is not supported. -The alias `--experimental-default-config-file` is equivalent to -`--experimental-config-file` without an argument. -Node.js will read the configuration file and apply the settings. The -configuration file should be a JSON file with the following structure. `vX.Y.Z` -in the `$schema` must be replaced with the version of Node.js you are using or -`latest-vX.x` for the latest version of that major release line. - -```json -{ - "$schema": "https://nodejs.org/dist/vX.Y.Z/docs/node-config-schema.json", - "nodeOptions": { - "import": [ - "amaro/strip" - ], - "watch-path": "src", - "watch-preserve-output": true - }, - "test": { - "test-isolation": "process" - }, - "watch": { - "watch-preserve-output": true - } -} -``` - -The configuration file supports namespace-specific options: - -* The `nodeOptions` field contains CLI flags that are allowed in [`NODE_OPTIONS`][]. - -* Namespace fields like `test`, `watch`, and `permission` contain configuration specific to that subsystem. - -The configuration file can target a specific Node.js major version with -`nodeVersion`: - -```json -{ - "nodeVersion": 25, - "nodeOptions": { - "watch-path": "src" - } -} -``` - -To keep multiple version-specific configurations in the same file, use the -`configs` array. Node.js will use the first entry whose `nodeVersion` matches -the current Node.js major version: - -```json -{ - "$schema": "https://nodejs.org/dist/latest-v26.x/docs/node-config-schema.json", - "configs": [ - { - "nodeVersion": 25, - "config": { - "$schema": "https://nodejs.org/dist/latest-v25.x/docs/node-config-schema.json", - "nodeOptions": { - "watch-path": "src" - } - } - } - ] -} -``` - -When `configs` is used, the top level may only contain `$schema` and -`configs`. Each `configs` item must define an integer `nodeVersion` and an -object `config`. A single top-level config does not require `nodeVersion`, but -if present it must match the current Node.js major version. - -When a namespace is present in the -configuration file, Node.js automatically enables the corresponding flag -(e.g., `--test`, `--watch`, `--permission`). This allows you to configure -subsystem-specific options without explicitly passing the flag on the command line. - -For example: - -```json -{ - "test": { - "test-isolation": "process" - } -} -``` - -is equivalent to: - -```bash -node --test --test-isolation=process -``` - -To disable the automatic flag while still using namespace options, you can -explicitly set the flag to `false` within the namespace: - -```json -{ - "test": { - "test": false, - "test-isolation": "process" - } -} -``` - -No-op flags are not supported. -Not all V8 flags are currently supported. - -It is possible to use the [official JSON schema](../node-config-schema.json) -to validate the configuration file, which may vary depending on the Node.js version. -Each key in the configuration file corresponds to a flag that can be passed -as a command-line argument. The value of the key is the value that would be -passed to the flag. - -For example, the configuration file above is equivalent to -the following command-line arguments: - -```bash -node --import amaro/strip --watch-path=src --watch-preserve-output --test-isolation=process -``` - -The priority in configuration is as follows: - -1. NODE\_OPTIONS and command-line options -2. Dotenv NODE\_OPTIONS -3. Configuration file - -Values in the configuration file will not override the values in the environment -variables, command-line options, or the `NODE_OPTIONS` env file parsed by the -`--env-file` flag. - -Keys cannot be duplicated within the same or different namespaces. - -The configuration parser will throw an error if the configuration file contains -unknown keys or keys that cannot be used in a namespace. - -Node.js will not sanitize or perform validation on the user-provided configuration, -so **NEVER** use untrusted configuration files. - -### `--experimental-default-config-file` - - - -> Stability: 1.0 - Early development - -This flag is an alias for `--experimental-config-file` without an argument. -If present, Node.js will look for a -`node.config.json` file in the current working directory and load it as a -configuration file. - ### `--experimental-dtls` + +* `fn` {Function} A mock function created by the test runner's mocking API. +* `times` {integer} The expected number of calls. +* `message` {string} Optional error message. + +Asserts that the mock function `fn` has been called exactly `times` times. + +```js +test('mock was called twice', (t) => { + const fn = t.mock.fn(); + fn(); + fn(); + t.assert.callCount(fn, 2); +}); +``` + +#### `context.assert.called(fn[, message])` + + + +* `fn` {Function} A mock function created by the test runner's mocking API. +* `message` {string} Optional error message. + +Asserts that the mock function `fn` has been called at least once. + +```js +test('mock was called', (t) => { + const fn = t.mock.fn(); + fn(); + t.assert.called(fn); +}); +``` + #### `context.assert.fileSnapshot(value, path[, options])` + +* `fn` {Function} A mock function created by the test runner's mocking API. +* `...args` {any} The expected arguments. + +Asserts that the most recent call to the mock function `fn` received arguments +deeply and strictly equal to `args`, using the same comparison as +[`assert.deepStrictEqual()`][]. + +```js +test('mock was last called with arguments', (t) => { + const fn = t.mock.fn(); + fn(1); + fn(2); + t.assert.lastCalledWith(fn, 2); +}); +``` + +#### `context.assert.nthCalledWith(fn, n[, ...args])` + + + +* `fn` {Function} A mock function created by the test runner's mocking API. +* `n` {integer} The 1-based index of the call to check. +* `...args` {any} The expected arguments. + +Asserts that the `n`th call to the mock function `fn` received arguments +deeply and strictly equal to `args`, using the same comparison as +[`assert.deepStrictEqual()`][]. `n` starts at `1`, so +`t.assert.nthCalledWith(fn, 1)` checks `fn.mock.calls[0]`. + +```js +test('mock was called with arguments on the second call', (t) => { + const fn = t.mock.fn(); + fn(1); + fn(2); + t.assert.nthCalledWith(fn, 2, 2); +}); +``` + #### `context.assert.snapshot(value[, options])` + +* `source` {Buffer|TypedArray|DataView} The data to mask. +* `mask` {Buffer|TypedArray|DataView} The 4-byte masking key. +* `output` {Buffer|TypedArray|DataView} Where to write the masked data. +* `offset` {integer} Byte offset in `output` at which to start writing. +* `length` {integer} Number of bytes of `source` to mask. + +XORs the first `length` bytes of `source` with `mask`, repeated, and writes the +result to `output` starting at `offset`: byte `i` of `source` is XORed with +byte `i % 4` of `mask`. This is the masking operation that WebSocket clients +apply to every frame payload they send ([RFC 6455, Section 5.3][]), and that +servers undo on every frame they receive. Applying it twice with the same +`mask` restores the original data. `source` is not modified, unless it shares +memory with `output`. + +All arguments are treated as raw bytes, whatever the view type. `source` and +`output` may be the same view or overlapping views over the same memory; the +result is the same as if `source` had been copied first. + +An error is thrown if `mask` is not exactly 4 bytes long, if `length` is +greater than `source.byteLength`, if `offset + length` is greater than +`output.byteLength`, or if `output` is backed by an immutable `ArrayBuffer`. +Nothing is written in that case. `source` and `mask` may be backed by an +immutable `ArrayBuffer`. + +```mjs +import { Buffer } from 'node:buffer'; +import { websocketMask, websocketUnmask } from 'node:http'; + +const key = Buffer.from([0x37, 0xfa, 0x21, 0x3d]); +const payload = Buffer.from('Hello'); + +// Write a masked copy of the payload after a 6-byte frame header. +const frame = Buffer.alloc(6 + payload.length); +websocketMask(payload, key, frame, 6, payload.length); +console.log(frame.subarray(6)); +// Prints: + +const received = frame.subarray(6); +websocketUnmask(received, key); +console.log(received.toString()); +// Prints: Hello +``` + +```cjs +const { Buffer } = require('node:buffer'); +const { websocketMask, websocketUnmask } = require('node:http'); + +const key = Buffer.from([0x37, 0xfa, 0x21, 0x3d]); +const payload = Buffer.from('Hello'); + +// Write a masked copy of the payload after a 6-byte frame header. +const frame = Buffer.alloc(6 + payload.length); +websocketMask(payload, key, frame, 6, payload.length); +console.log(frame.subarray(6)); +// Prints: + +const received = frame.subarray(6); +websocketUnmask(received, key); +console.log(received.toString()); +// Prints: Hello +``` + +## `http.websocketUnmask(buffer, mask)` + + + +* `buffer` {Buffer|TypedArray|DataView} The data to unmask, in place. +* `mask` {Buffer|TypedArray|DataView} The 4-byte masking key. + +XORs every byte of `buffer` with `mask`, repeated, in place: byte `i` is XORed +with byte `i % 4` of `mask`. This is equivalent to +`http.websocketMask(buffer, mask, buffer, 0, buffer.byteLength)`, and is +typically used to unmask a received WebSocket frame payload +([RFC 6455, Section 5.3][]). See [`http.websocketMask()`][] for an example. + +An error is thrown, and nothing is written, if `mask` is not exactly 4 bytes +long or if `buffer` is backed by an immutable `ArrayBuffer`. + ## Class: `WebSocket` + +* `source` {AsyncIterable|Iterable} whose chunks must be {Uint8Array\[]} +* `options` {Object} + * `signal` {AbortSignal} + * `limit` {number} Maximum number of bytes to consume. If the total bytes + read exceeds limit, an `ERR_OUT_OF_RANGE` error is thrown +* Returns: {Promise} Fulfills with `undefined`. + +Read a source to completion, discarding every chunk. Unlike the other +consumers, `dump()` retains nothing. Memory tops-out at a single batch no +matter how much data the source produces. + +Use this to consume a stream when the content doesn't matter. For example, A +QUIC stream only returns flow-control credit to the peer as its data is +consumed, so a receiver that does not want the payload must still read it to +completion. + +If the source errors part-way through, the returned promise rejects with that +error. + +There is no default `limit`. `dump()` reads until the source is exhausted +unless a limit is specified. When a limit is configured and the source exceeds +it, the promise rejects and the source is cancelled. A partial dump is never +reported as success. + +```mjs +import { dump, from, pull, tap } from 'node:stream/iter'; + +// Count the bytes flowing through a stream without retaining any of them. +let bytesSeen = 0; +const counter = tap((chunks) => { + for (const chunk of chunks) bytesSeen += chunk.byteLength; +}); + +await dump(pull(from('hello world'), counter)); +console.log(bytesSeen); // 11 +``` + +```cjs +const { dump, from, pull, tap } = require('node:stream/iter'); + +async function run() { + // Count the bytes flowing through a stream without retaining any of them. + let bytesSeen = 0; + const counter = tap((chunks) => { + for (const chunk of chunks) bytesSeen += chunk.byteLength; + }); + + await dump(pull(from('hello world'), counter)); + console.log(bytesSeen); // 11 +} + +run().catch(console.error); +``` + +### `dumpSync(source[, options])` + + + +* `source` {Iterable} whose chunks must be {Uint8Array\[]} +* `options` {Object} + * `limit` {number} Maximum number of bytes to consume. If the total bytes + read exceeds limit, an `ERR_OUT_OF_RANGE` error is thrown +* Returns: {undefined} + +Synchronous version of [`dump()`][]. Throws `ERR_INVALID_ARG_TYPE` if `source` +is not synchronously iterable. + ### `text(source[, options])` * `socket` {stream.Duplex} +* Returns: {boolean} Called when `socket` is detached from a request and could be persisted by the -`Agent`. Default behavior is to: +`Agent`. The `Agent` calls this method only when `keepAlive` is enabled. -```js -socket.setKeepAlive(true, this.keepAliveMsecs); -socket.unref(); -return true; -``` +By default, TCP keep-alive is enabled and the socket is unrefed. A server +keep-alive timeout hint may adjust the socket timeout or cause this method to +return `false`; otherwise, it returns `true`. This method can be overridden by a particular `Agent` subclass. If this method returns a falsy value, the socket will be destroyed instead of persisting From 5fd03e6fdf5bfc79219ead914deb28541c2183e2 Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Thu, 8 Oct 2026 17:58:31 -0300 Subject: [PATCH 49/98] child_process: propagate config-file permission flags to children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66559 Reviewed-By: Marco Ippolito Reviewed-By: Ulises Gascón --- lib/child_process.js | 2 ++ .../permission/child-process-inherit-test.js | 16 ++++++++++++++++ .../config-child-inherit-allow-only.json | 9 +++++++++ .../permission/config-child-inherit.json | 11 +++++++++++ 4 files changed, 38 insertions(+) create mode 100644 test/fixtures/permission/child-process-inherit-test.js create mode 100644 test/fixtures/permission/config-child-inherit-allow-only.json create mode 100644 test/fixtures/permission/config-child-inherit.json diff --git a/lib/child_process.js b/lib/child_process.js index 32317b6621c4..ac7188048be6 100644 --- a/lib/child_process.js +++ b/lib/child_process.js @@ -574,9 +574,11 @@ function copyPermissionModelFlagsToEnv(env, key, args) { } const flagsToCopy = getPermissionModelFlagsToCopy(); + const copiedFlags = new SafeSet(); for (const arg of process.execArgv) { for (const flag of flagsToCopy) { if (arg.startsWith(flag)) { + copiedFlags.add(flag); env[key] = `${env[key] ? env[key] + ' ' + arg : arg}`; } } diff --git a/test/fixtures/permission/child-process-inherit-test.js b/test/fixtures/permission/child-process-inherit-test.js new file mode 100644 index 000000000000..2a095b7d1424 --- /dev/null +++ b/test/fixtures/permission/child-process-inherit-test.js @@ -0,0 +1,16 @@ +const { spawnSync } = require('child_process'); +const { status, stdout, stderr } = spawnSync(process.execPath, [ + '-p', + `JSON.stringify([ + typeof process.permission, + process.permission.has("fs.read"), + process.permission.has("fs.write"), + process.permission.has("child"), + process.permission.has("worker"), + ])`, +]); +console.log(JSON.stringify({ + status, + stdout: stdout.toString().trim(), + stderr: stderr.toString(), +})); diff --git a/test/fixtures/permission/config-child-inherit-allow-only.json b/test/fixtures/permission/config-child-inherit-allow-only.json new file mode 100644 index 000000000000..7f26f3d672ee --- /dev/null +++ b/test/fixtures/permission/config-child-inherit-allow-only.json @@ -0,0 +1,9 @@ +{ + "permission": { + "allow-child-process": true, + "allow-worker": true, + "allow-fs-read": [ + "*" + ] + } +} diff --git a/test/fixtures/permission/config-child-inherit.json b/test/fixtures/permission/config-child-inherit.json new file mode 100644 index 000000000000..c4d4f1a25152 --- /dev/null +++ b/test/fixtures/permission/config-child-inherit.json @@ -0,0 +1,11 @@ +{ + "permission": { + "permission": true, + "allow-child-process": true, + "allow-worker": true, + "allow-fs-read": [ + "*", + "/nonexistent/dir with \"quotes\" and \\backslash" + ] + } +} From 846236367dbd6090ad168e641971b3d6d7b63e5f Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Thu, 8 Oct 2026 17:58:43 -0300 Subject: [PATCH 50/98] doc: document URL and Uint8Array references in process.permission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was missed in https://github.com/nodejs/node/pull/65492. Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66557 Reviewed-By: Luigi Pinca Reviewed-By: Ulises Gascón --- doc/api/process.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/doc/api/process.md b/doc/api/process.md index a3055851ebdf..af6875dfa6d5 100644 --- a/doc/api/process.md +++ b/doc/api/process.md @@ -3139,10 +3139,14 @@ for the current process. Additional documentation is available in the * `scope` {string} -* `reference` {string} +* `reference` {string|URL|Uint8Array} * Returns: {boolean} Verifies that the process is able to access the given scope and reference. @@ -3156,6 +3160,10 @@ permission status, but denied operations will not throw `ERR_ACCESS_DENIED`. The reference has a meaning based on the provided scope. For example, the reference when the scope is File System means files and folders. +For File System scopes, a `URL` reference must use the `file:` protocol and is +converted to a path, while a `Uint8Array` reference is treated as the raw bytes +of a path. Other scopes ignore the reference. + The available scopes are: * `fs` - All File System @@ -3164,6 +3172,10 @@ The available scopes are: * `child` - Child process spawning operations * `openssl.store` - Loading keys through OpenSSL STORE loaders * `worker` - Worker thread spawning operation +* `net` - Network operations +* `inspector` - Inspector operations +* `wasi` - WASI operations +* `addon` - Native addon operations * `ffi` - Foreign function interface operations ```js @@ -3176,13 +3188,19 @@ process.permission.has('fs.read'); ### `process.permission.drop(scope[, reference])` > Stability: 1.1 - Active Development * `scope` {string} -* `reference` {string} +* `reference` {string|URL|Uint8Array} Drops the specified permission from the current process. This operation is **irreversible** — once a permission is dropped, it cannot be restored through @@ -3223,6 +3241,7 @@ The available scopes are the same as [`process.permission.has()`][]: * `inspector` - Inspector operations * `wasi` - WASI operations * `addon` - Native addon operations +* `ffi` - Foreign function interface operations ```js const fs = require('node:fs'); From 4ce8412e18ba7371d9462c7223b9279bf784d546 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Mon, 5 Oct 2026 10:32:13 -0300 Subject: [PATCH 51/98] benchmark: fix url-searchparams-sort to sort a fresh list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copying from another URLSearchParams only slices its internal list, so every iteration sorts a fresh, unsorted copy. Refs: https://github.com/nodejs/node/issues/50566 Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66535 Reviewed-By: Luigi Pinca Reviewed-By: Ulises Gascón --- benchmark/url/url-searchparams-sort.js | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/benchmark/url/url-searchparams-sort.js b/benchmark/url/url-searchparams-sort.js index fcd55cd39795..2b40085e3679 100644 --- a/benchmark/url/url-searchparams-sort.js +++ b/benchmark/url/url-searchparams-sort.js @@ -16,30 +16,17 @@ const inputs = { 'k&f&j&i&l&m&g&j&d&i&z&q&p&x&q&q&d&n&y&w&g&i&v&r', }; -function getParams(str) { - const out = []; - for (const key of str.split('&')) { - out.push(key, ''); - } - return out; -} - const bench = common.createBenchmark(main, { type: Object.keys(inputs), n: [1e6], -}, { - flags: ['--expose-internals'], }); function main({ type, n }) { - const searchParams = require('internal/url').searchParamsSymbol; - const input = inputs[type]; - const params = new URLSearchParams(); - const array = getParams(input); + const base = new URLSearchParams(inputs[type]); bench.start(); for (let i = 0; i < n; i++) { - params[searchParams] = array.slice(); + const params = new URLSearchParams(base); params.sort(); } bench.end(n); From 9ef698c7818c185c6fa7971aaf6010a6608cd8a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Ara=C3=BAjo?= Date: Fri, 9 Oct 2026 12:53:53 -0300 Subject: [PATCH 52/98] sqlite: validate aggregate() name and options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Database.prototype.aggregate()` cast its arguments without checking their types, so a missing `options` leaked a raw V8 TypeError and a non-string `name` was silently coerced. Throw `ERR_INVALID_ARG_TYPE` like the other sqlite bindings do. Assisted-by: Claude Code Signed-off-by: Guilherme Araújo PR-URL: https://github.com/nodejs/node/pull/66463 Reviewed-By: Trivikram Kamat --- src/node_sqlite.cc | 13 +++++++++++++ .../test-sqlite-aggregate-function.mjs | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/node_sqlite.cc b/src/node_sqlite.cc index 2ec54f73693f..6ad797a520cb 100644 --- a/src/node_sqlite.cc +++ b/src/node_sqlite.cc @@ -2703,6 +2703,19 @@ void Database::AggregateFunction(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); THROW_AND_RETURN_ON_BAD_STATE(env, !db->IsOpen(), "database is not open"); THROW_AND_RETURN_IF_IN_AUTHORIZER(env, db); + + if (!args[0]->IsString()) { + THROW_ERR_INVALID_ARG_TYPE(env->isolate(), + "The \"name\" argument must be a string."); + return; + } + + if (!args[1]->IsObject()) { + THROW_ERR_INVALID_ARG_TYPE(env->isolate(), + "The \"options\" argument must be an object."); + return; + } + Utf8Value name(env->isolate(), args[0].As()); Local options = args[1].As(); Local start_v; diff --git a/test/parallel/test-sqlite-aggregate-function.mjs b/test/parallel/test-sqlite-aggregate-function.mjs index f588cc3ee680..83f7833cb3f5 100644 --- a/test/parallel/test-sqlite-aggregate-function.mjs +++ b/test/parallel/test-sqlite-aggregate-function.mjs @@ -20,6 +20,24 @@ describe('Database.prototype.aggregate()', () => { return fn; } + test('throws if name is not a string', (t) => { + t.assert.throws(() => { + db.aggregate(123, { start: 0, step: () => {} }); + }, { + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "name" argument must be a string.' + }); + }); + + test('throws if options is not an object', (t) => { + t.assert.throws(() => { + db.aggregate('sum'); + }, { + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "options" argument must be an object.' + }); + }); + test('throws if options.start is not provided', (t) => { t.assert.throws(() => { db.aggregate('sum', { From 714aca1ef6c5d7d0f4fd8989657f4b9584323df3 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Sat, 10 Oct 2026 01:50:12 +0200 Subject: [PATCH 53/98] tools: switch to PGP and SHA256 in ICU updater Signed-off-by: Antoine du Hamel PR-URL: https://github.com/nodejs/node/pull/66469 Reviewed-By: Filip Skokan Reviewed-By: Steven R Loomis --- .../maintaining/maintaining-icu.md | 16 +++++- tools/dep_updaters/icu.kbx | Bin 0 -> 31790 bytes tools/dep_updaters/update-icu.sh | 52 +++++++++++++----- tools/icu/current_ver.dep | 2 +- 4 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 tools/dep_updaters/icu.kbx diff --git a/doc/contributing/maintaining/maintaining-icu.md b/doc/contributing/maintaining/maintaining-icu.md index e83aa8a5b0c4..93dbda2c7d95 100644 --- a/doc/contributing/maintaining/maintaining-icu.md +++ b/doc/contributing/maintaining/maintaining-icu.md @@ -177,9 +177,19 @@ make clean tools/license-builder.sh ``` -* Update the URL and hash for the full ICU file in `tools/icu/current_ver.dep`. - It should match the ICU URL used in the first step. When this is done, the - following should build with small ICU. +* Verify the PGP signature of the full ICU file: + +```bash +gpgv --keyring tools/dep_updaters/icu.kbx \ + icu4c-*-sources.tgz.asc icu4c-*-sources.tgz +``` + +* If the release was signed with a key not present in that keyring, update it + from the [ICU `KEYS` file](https://github.com/unicode-org/icu/blob/HEAD/KEYS) + after confirming the new key out-of-band. +* Update the URL and SHA-256 hash for the full ICU file in + `tools/icu/current_ver.dep`. It should match the ICU URL used in the first + step. When this is done, the following should build with small ICU. ```bash # clean up diff --git a/tools/dep_updaters/icu.kbx b/tools/dep_updaters/icu.kbx new file mode 100644 index 0000000000000000000000000000000000000000..6abb79eaaf374f7445926b97f61f16b100b8f01b GIT binary patch literal 31790 zcmb@u1zc54*EqUQ(cR6VySuwnx}>|i1pz@oLJ$y15dmqXlvYAO5CkNpQ9wZHl(-w8 z$LD?j-}m0{e)sYesa(zoybo04^|4N8F zDC9R30RRXL018kI@;&?zEU!>teBi^aI2^VgZTajYNP)bex}2a+5$qVuhx&ZxSi<)s zqDFbu38E!$%D@Bxu)wSUgsgsFLjm~j@Bfnh;ou1b8?O0Dwikj12QX>+NFT-8m~sP&)Ez4d3V*hS`LOUoW)FxqW1@VQ^1lAS}jSPM+-Sb zxkH<{{X_68SJ@-(1wmWZT$0pvN0}3y>Rep*rY_Nf?n&uycFRsm9k;CYq2!}%kC&Kh zj70^NIV2-B%--XS(9ZR=-L0ZLx_kT6olp-+6W;Th4Nd57@3I%!U~@TzlTp0x73z5O zxOR2ta=kU?XKH*)jF55l=(Kp!2cbe8uM^n(2YSMJR5?)~c&Z<5cUAjObXcjdBf0pK zNpoMR90t!6lZ)FMoLLo}PPiziIB^(=VXikOI(zqCEp~L1L#Y9L2mpBr_MVfs(_iBO zG+3!A7$S9jy<8pa{E@P1@>)n%HBWyBUr!r<7cWm6ccg-shqsrfgQveA($UKoY3S)< z=VkAJRCEY*aQE`|01J>d{z$MQ8&cB6F2Kdp-(5DqkK4u8gWJx_L;COALH`<^1c)e{ z5CR405zw14fl|UyFwxOaVNeueG$=X-CY%HcgFtB@P|~{q20Gp!&6vB7dP((b?RxFw z*|#@mGB3FW2)23Yxwlm9;q${wckLPSMqOzu{aIx*Uk)OLH$&4O4G5SZi^p@r1Q@Yi z$Mm&fHk21`hBMgo3fC=7u9Su{G`i=o*7T*x1s*o!zKnkI98xkl#B}Wd(fNR=XYwsj zR8ifB@00K)z0)jwe*Z0*4AqFr6(k-S|0?OoaCSv##A0QxxF=F;d0~9~=ZPi0$>)bi zbF{P3x{tGclwZuK3e?AHjuM%*@mxs|h!V69ZSJA{Ij19uY5S>ncDQ+y&g~J(c;k zZ8hqfYwCQ%MCj-o=Zvp{+eNSyxT%WYn4*_OLOUR8aYuyZa*jS$bEfGpQ2-b?Zb0+D z^m#UD%^its%#uw$C$+CLG3X^7`dKQunfWf*m|8RCW2IaaN2giU_~(J}otmCQ+4gE$ zb~Y(V(s3s@ zR`(@4E74NHW#g@C?17iy54CtHbxq6HUb1FlZm+HC*>bT|>%XuTRK!j+TTF<`r~3N+ zrOycvS~wY~Qs{qF3hUoBCl85wg%mHk?KC8QXx+29_E?NiX2t1vC*Er@m-+Om zSR5)HDo^>{@Th|GI__V>Y$GgU9z7=ID?33CzkZHfFxOdrRc3Ttp=SzL?zQWA`5^~N zRv9|a(sb+oEGy@`f-&n!OyRNC0}L0EidgDBuYB`HiL7f)8d0kNoP|Me$7WZOW=jBo zzXYu-m1brhu0hz?^y?UN(u(q$JY4LrcfiLN`L}`nZGWMlB?rWQV<7+C{{CZoq1bQ( z^7&Z^9#Ec#ebGn=6Y@2%c%K`_vnN#@P1efMAn%| zRGsrbX5wN!8h_qhuj(kqQmyRSQm`2Jv)b0~bm%q%v**Vy;-6T;u@1^1fto(1W_;<@ z`X*`LKZH@NvZQq%4>aZ73MmQ3lBM7m)wWI;`9hnm8A`pga#qw!a7K0(f0%guwT<1H zIm=$6MU#gai{I|R5s!C!hrsNX{*~cwiG|ARH$)5Fn-kqqooM`!FB94a1ismjks1iM zmO_c^CD+Gi>-S`5q-!ziR+HirF%{xhwDVdQKc^W;jvDk*HZf_o+DYvA(ZwAeOL!5% zy$&$n)d(*+q-h2O+u;^bUB#RQ^Jj48FILfZL4pr95W$kO$1q$;QIr?C4eR5pcyjG{ zZy+R!6GeAbLHPnS0- zTw(s?wO#S~DAmyRCXZvKtFB;A`1@cF=%XbS4L76?5p-wd-ctUYn-d1^1ACNgLn&OI zs>1&?_lW8HgR_VyQjZ&{>E-3&;)i5a&{0%E>Z_?}t7)qsHIz)jNyX3CT^5{JxV-zZ1^SWAm6 zQsfXL5laLm=3;u_A#sCep7>5@wvJWO3I3a&=-Kzh^FI>ccyipKdE8EZFG z@*xxj5q4Xii5c-)8E-(UtIP1rtz5nHlgcuIjz8K0z?h8I`qLr7zIPWPiG@z|78QLX zd*f;_%cF|Q7p=Ljqu+eR5qXp_5(r2HX6Xxlw_u59Y{K}+*cT)FxSb*B*Cguc<+)C3 zDTxxymgo(wV1@sDmGV`--lyX`1iTiA+!1pEFSOX82CmNQ6iZW&hm|g8htqEJ>Ln5j z;wj;BV?Xg1vZ>Wohw1Pq;)ux%B6S{@d^~dAVqr@$Y8{c?`7*=9 z(%rGe=PM?ur;SY^XPGFAottimGro#ZinBHYxbCOy;G_d=ZYNeT^>!*`-<6QEe}*mn zfX3)^G@7bM1bB`|O_VL`5A7MF?M|*{2(|9OWz@$h4BsT4d_RT0uub~vl%~by-LS>k zrCN$PF5~&M{i2!XMMq9e?LShd5Uj zLrukEW67dVr0fH}=R5|J&Al;@@fA}nyry%Jne*G?P> zLSKem0N@Gx+d@VI0Qa{W@AWYsiTm`5cudFo`|+xhFUyyym3yEqMBdm#ED;|zd0h

3BA=aZE(B1#WYm$s8_Y2+Y*JLN`k7@NQKTW3Hs~HCB#I|~OU8y$ zd!V`tVru_@`MZtp36E(4?0(y)Sta9Gl%E%c-6=R!#wnY&EEOtzM$qSd7(s7I<{tSd z4$zV7$d2o5;ZdaoMj&V#J5zh`kLnAVI5BfqlIthQ`?C)s5!6hM*~zJeUPCjkPXa$nkp2w2DaZsmW>E@X5B*zyBvIBKY5}$iMlt z|2qQ$`P+c}PYx*pYC7R3hd{>~xl+GzG5!oZYZU6WaxBFy?Iy%Wsv?x_+m6O$b|Q2R z_R`wdU&5cJh_~N;YIwm}_|%=s%ED)=V*@aF4rEUqG)w z)B5t!Uc61|EcOtm)h9zc)D2zKMwUygwbmHN=WCqn++l?jp0-n?%zIhKis*Fd=ZK_{Kf6h12lRK0Ic~pSvEN5gNwp{OV`ua zO8}m_vYIkj00C6NKj330oEEE{iccPJhJczF8z>Z!;hO-#*@ zsPh0OfBS8~7^- zv)cY!J5K)w+t}Ft)25A$(?9TEvVb+g8!Ni_dAr+${J!}AGxKl{0Q>8Aq62d*XJ1u4 z@S6ziw>&p5jT`u8$^O#QR$B*z2|&2b1(fJ-c-T2W)ewYl=$GvL6u`a&NI>||-bPUa zgsA`kiV@&uDEBwaY~u?;zv-d!F8*o;f5S$;UV4AG4|Vl)*S^sYupMZ)vxC|nINiZd zNgu?6JV0F?U6j>87#u6;i%@3+V-N=UguV%MG13NM91!mDbJM@!;ijFjP-lf3{BIiH z0KFTY8A1Gpqp$Lfj35mR%fk=ibPyn5Mmhq7n#$_l21_ zcpCoA56s!yU;d_@8~RXh_uu*r@(D|Ka96p((}VDHzd-#zb>I8@8r(K^nh*>dF5ubpQfE{N+E`zv^-Uj=xI-f7fsW*1$V>z$xI;a0$33 zn9G76ez+)H>`y%;?jJr;lu#5=R8f>s*#2MuJCJL4@Z$@({0E!{Smppq=L_bZe`NSq zdx$l}C}IUMfmlM!07k&q->v*Dsg+B#?;j5SX;}+c0$cm5R~*0|^8~*Cp@B1lTnoYF z;Zh*~An$MzI0e837X&4dgUf)j34z!@y?4WPz(2hGt1tib`yaX+u95#H2s6*!WAdg8yf@~;vXz#gR1_^*_(cvwBG5B3f=0BeK229U5$*c;doto;W5n`hCeQ76p#gvcz2CMPLNf9%oNFlPF;#$e5b-aZvo`=#oB{yt z>wn=Gw!y~*T)0;I*ai3o{?#7*<_iH}fO7>AKn~CV3;-)A6F(pfNC2{c5}*#~00w|5 zUNkbTHG6b8kD5<#h<%upVvC{zxr0X2Z$hB`rgprO!v&=hDk z^cl1Y+5~+E9fVFmm!MnFBNza~gpt7LU|cX!m;y`(W(ISF`NAS#iLfl#Ggu9*1sv;f z*b;0Tb_Pd*6T*>jE>J&J;f8QKxDPx6o&?WMZ-j+MdL?PMl(b6M2kktLaRXQKpR6_NBf12 zht7m9j;@1lj~;@aivAqE1$_kl8~QH{d<+&0DGUP)SBxl(Y>XO=UW_@6158XzdQ36Q zTbM4GcQLauUt;!SE@Pfy5n!=nDPUP(1z@FMm0`Wbn!!52#>QsGmc=&1_Qy`aF30Z1 zUcf%ZA;jUvQNyvviNwjpX~G%9`H72;%Y-Y3Yl$0*n}u7CJA%87hmOaLr+{aJ7lD_H z*NiuXcZ5%f&xfyr?}49$Ux7b>zd?XXz)YY-U{7$5pqQYWV3iO?$Uvw-Xh#@B_?)np z@EZ{d5etzTkt@*yqH3aHqFrJFVnJdf;$Y%j;&$RC5*P_Hi5iJJNh(Pl$t1}+DK)7a zsRL;OX$|QZ=?NJnnJk$DSt402*#y}+Ig(t7+>Jbq{5APJ1)PGN;ub{^MFB-G#Wp1o zr39rNWg=xA5j}s_8@oZsOVJb0_ck9hUtFMv(OvTN7L8R&of{!h%z`bWH59w z>@w0aYB7d0Rx-{qp)-jxIWs+G>SH=#W??pFzR&!cd4q+FMU5qtrIKZi6`NIt)ra*N z>t{9;Hc>V=wtTiv>`-r2JZ3y;JbgS@yh6P0yw7>R@DcE-^F{Nu@a^-n^V{*~@_!b<7El(5 z6nG=BC&(e_AXp$cEkr1!C3Ih?OXyNqMA%ojM)6M+7 zE0uRugjB**-l`&0HB{47r_`v`?9|HDcGX4H@2K}^U}@-U>n!-Ll;Z*UHAK-Wp+TY+Ye}VWVU7%;v~e%{I?=*G|FivE7!vtbK<4cLynl zbcc0EDaUlj4JRq5M^4|JWt_8|w_Oxma$NRYRb2~RPuz6e%G|EqjooWK&^)X?nmq|T z-8_4|XuN{GM!h+_G+ zeK3BoXYf!6dq_gadZ=<}Nfu z%hTy1m7=C+%+IolQHleL*Pa_a?<)~4DKDiceNcK?=2^D%LhnUSxk!0=1yx0ACA2c2 z@_Us<)u(F3>Nho9HAS@~wGUnbF9Tj~zOsHbS*KCgRWDj!)46_7J_2#Y?iI&Dz?$)w4y0+YQ;`Y=IjE;L9*PWrAM{m8~ZohMSx7uaZ^`+aS z`*Y8&p5b1N-v0N>@4Nft`a1ii`r8J?2bw>Ket117JlHfOG}QP}=wsus@NmUaR=2+TeZBa0Zw-4bYn^tze1m_Z?Yr{#Pn+hO z-?qHBer?D7!2gl=ll5oAj_l6RuF3AVJ>R|S{iFl(gR(=x!>%LUqlIIS<6kFm3`A8%i4K8M~T9nr4|*n|E8x-2Q0=wI;D)vz4^dwzqNccZ_nn>m2P8<9g36 z&ON~+(KE>_#XHUCk#DBoWB=TM%s}5D-C(W|%#h8{{;<;UxI0b}s*&t>vG49hO-8rI z6yA%C^@+2)Zx!#95SCcaA%+tq|*|JX@b5e4f^S&2Q73w`r zF8c87^f^umbE!<3>5HK9?25L^Z`A}f3bny6U%lF@r)|(}jA*KSJ@MwUg|St=&8t1L zqrLOnJKQee?pr-Jy>{=7`Xu@Z`sW66KNt>T5B3g4e^egEA6^`({uDl{K1M#aK3@Mh zazbsAU~+cq$@Hxml$o|!|1aEMHs(s^Z5QYkmKF<_td?n)7gzFDjlSZ1?f&MshFlw5 z_uU`^HOhVyce86NV4HP&{zv*x)t}cpZ+89mc=opU-yM`6rXK|!o1Mss#0p&>ed=Q znlV}#+66l0x^=f&_1^1$GMF)3G1@dfF!^N$GsmxAKqZ~2bs?@HEtL~^de#!aDzAm?Zq7k=A<8{KD zz7|NUc$;^7amQ%q**i*5A2fR`dR^Z8_IdQ14Jdrz7$hA0HMH`vf4FAk(Wl^1%Q1y< z#?O$?^ApXJ8B<=<>N89;*RwNU>gJN>9T#L5DHl(cJ}9D0 z;d0}N;HwZA6M7QG5toqkk?xU`Q>apgP*qc}(o!N7>0IfP8D27uGwrfqveK~$vCDF( za;kHwaLe%s^Rn|%@uTp67Z?z%5K0#I644bE5F-*h5}%TIBbg%=E^R3zD@!kjkozt_ ztWd9*trVebr=qILsfMR^q&}n3rdgyFr|qJns>=fU3g7g4^q(0-8Cn`i7?T?xn~a;* znkAdNSjgX|ynSFfWcA!S%tp_a)%MD6+`il)!cpIe)#==M&?V2+!%fN^+kMWX(lf+M z$(z7?!Kd6ez)!;O+W%cZOrSy#G^jH;EQBANmvh6^!%oAi?)XHQMXKBtjG~Xmi@uKe zac?%ZFRuQ6etdj_ccSY9mn7$8=M zOTov&`KMb&C&j>X)DoIf{j$^-Llu~nT2*P)v$d=*LtYKnvoyps&b?-QHhZayR@#t?&_X5y069_s zVCnz>w^RTi`5pi;O#%QZ1_0<{1Au`T08Nwvph`3VBx&fM`2!P-&qx7g;94>hs0T)Y zT?ih87jg^Y3(0`gLIxolP!uR5Q~_!Sje! z7r@`ccMxOJb| zV%*1=!eqmY!d$?Tz{EZNm86p^En4nC3 z%*iaKtSW3G>>M0)oD^In+(bNtyu^HD{L}&rf}BF)!YU%SM2*Ds#FZp?BylCzq?)Af z$|%dC$@a?KmA|DRuPC6zqD-wqsEVluQ~RZUpz&REQR}mIzfP-e&82!;WW9mVSLD#&YHoWrI2l&b0@E~V5i8kc&JqL z#j{F`YTw$qI<^j~${eIqZVFG#`-+$W=Y=8xf*Ej+9!N|b}U>kx9#siGO)qf$R12PY} zg3^L3dwXa+xRM`*9>b`>Rl6%J9US>pI2K$OZUe5!-@v~i2oSOePec*oGYUG2G)e$U z4azzy1FAJ@A?hL;1DZ41OSE%z74$6h4RC$@2xA*l1+x+h7b_5J0b2#T9tVk&igS(Y zi@SzrjyI04iQh+{NYF*7OgKbjK(t8gMtnw+KuSgWicFPknLM0=f})wyjPjDIl-hs> zPV<`99m$AXpv$Ed*#p^~<;s*0kjoSL+{q=vYrsFtv{ zppJkp-z^?JF@0476GK;{aN|^yV$&wG0rNSFpO%0Xfi;7Tkgc+vslBJeJ;ws4X6GrF zV>dE)2@h+}IIk-2ai4QPdVlo*|G?s)i4crXg|Mjb-U$3i`@8R=nPW0yG2;^AaT4ku z_$CXcoImVNPtS13lFTN1a*{if-&xpF)LPtG(qA@FzFv7*L-ta*&a@$->G_+XmZNsM z&Rg$dx?6e=`vpG)4Rwz&k3JYbm^7Py_l0Rbe)0Q?`qzqejPE>K+&@TuuJ2atI~{Ty z!%yMo3Ky%t$DaV4JXi_B7UMV;s6o~k}gsYFqSYx9!Vigu|ruw zL%%t>f0OC89|K= zOkS8?o10rSTT)xaTJPD~+O;^~IXXMNbD?q#bDQ^2^sMzp`aJd{^v?)54>Ag_3&jib z3jY)#6qz4|5gi_L80#B%6d#c={(u^sJ?c}gAF8E2N}tPM$aKl7dc5<5C&w$dCU3id zyD<9cmuKR|MJ0Hp@h>pTpH%WyeXhCtlK<6aeQBffYwkBEEpOW%bQr#+eYeqF+w0#a z(tq}$Whi8rd*thA?zq7O_GJHb%&hd6v-yTa&tZ0ivl!<(dArrT*hT7FLK ztnaSwecA6kczWo4BzJUu+AM ze=~ak5G;8x{J3y^eZUMlmFeJi#su$ zi<__iBV8{8ca?%&!1ark7QhY`p#ca80r<)oyvt3@6hZ;UId5P9P=`?d1M5Lp|E2-g zHTPY9zhJ(hxp}X0lWyKa++6?l4Jzh86cSJWdL4aJdh=ESxc&mhb)gW*%|8_U2NMhd zyGbZ;I1GV`0tWuT$RH{jCORq_1{w+qIyShA4-*Ru3l$w32OA3q1Tk+Kxv2~LT^E6g zhJpsh^8R0@>$dMbVfQ9MpfETBv z{rH5$^hX()S&y@yG`A3h~(3XUcEzS5YXEtw)GvwAYl|-X4<Y97Qg~m z+faNMJ|F{JTmyUIC)gey3D1EKk?{w8lP#C*!z&It3;n)gXUVp~wW|DPv)+-Dv|l+! zg#wyRLzcplolKJ&jMAcCMZ+}LA-YaU>57Xl;4*i>BXV8&}R|bFp{>ES$-l1{Bmj!St#>a z>70dIXoHM_jc0j(^H@f`>yy_XEN1tGjCk}^IElIA9jgaEz;>DdIXlt3Xh(*F8;Kv;=_Nho!a#7Q(IU^1U$Gg`a( zb(x9N-mF|F%Prtab3dfFM>}YM?m~-Y$2==XHgZ8|GWb!qjvE6(yZFj0>GvC|>^QhS z#&JErl>BY<9&;yS);KfHe*cPNW^^Urf;(H-rMLJ!f?#wIt=iMlyfs?o!!ogpN(m;zhA0#f@6nBB(JE1quX3ZQUY?5E2)Dn_N{Ny zCshr~`M3994eyGrMoWpEwBxGYS0DF((%~wc2hgI(MoHAA5B=l<%2H>Ia|MtgH3D-}2bGwE7y{*TaK#r_g8Sq)x6k zWUfU5-2#x_`(b&uN2!hTlNZp|b{p6c%o5S_z))Vbf~2qHm2Z9&z_^u^(5CNE$3?^~ z?P3y9ew+3O+wL=L{&Bs%*hCA=U-<&RI0BFMZ#krM^|^ebPQ=X)prVOx-XXH&7UY%B z8;U$+thGDcb1mRF#jP}DL+5-&ryVN2_xh+em~6<<0zn0PQ1>Y3@u|hcNbPcH)BDLp zwFBSUcIQidws7S==mpIwLv`@uTlJ6iQk7nYb+S{zBr1|vrRv}KeS+ki?({@9IQ4iX z(XeV9fm=-Kcx>WLdVO3tRgdk0C7%6~=Pw65xEj7X?e1L3T+O%_*4tl^w?~_6`EEI6 z6~QZ`i;UM9UKx>XZJAirll~|w9$cRKbj2|vNcb-6b!!|Q_mrphH<|p1!Y)x48~$F6 znT7q6+Imal(GrKR<5E;Ryk&_WiN&WMqrK=b4>|}dYdP~5PbG_(B)kyZqnx?W8sBjo z_xLfYxbmKsT}=*a**-aeE{cKWu5vK7^sCdLs_CsOcvHvjz8ZDr80T(PLc_fPtY(_B zbK{=K#=;+{OSSP=B;(;M%Xb7fDQX%+5RLOE(^=EzUE}Q3iDHZ~p8|F!J64<1rC8)r z7H0W8;rR55qIl1cN>()}I?dJLN||Wiyt`Sdo;Mq31f6klUySY(`RJD?RuXeNS9^KZ zmK~L(u=IV5K9zjEu)fe%vCkM*jv1BTCq^zbf7+^hfO#-=;THYo;n_|%PAHVm3ir_` zgYxO+zRzoEA5Qy>p|_C|HsM`pGqs|H}THnG-xQ6A|0umk64*XtOyp%UVX)=!IFkb=+)% zk-L=A>%f)J`E4&uj>+z2BYL$rBEhvHcP<(>!aGP)ieqc_N@>2g3bA-h&bmefTum?5 zai*UZtZMIl%iSnotN^t;3`eJs?mv@Wx_a6vZ#fsN7R`xW_W;JAW z&g58hwmhLv9<1-zK^rDR$U}he9+J0vox~+4_Z!(b!_X7bmh&;he_6ZXG7~6KcNtJyG=JUHkuO0m&HP#9(QD!l zT0tE~$h1esNAs$7l7yd)#kdYak{cchjI~t}w=M_n{iiwlIo!EqQ1r=f+o*9lVYCZvah+@ za8q&$C$s8nW?@0TOrs=9nOL(9|-?eQ2L>z+nx zCO3XY92F{qZ&Z45$LZr`=Qh8h+(=T>(qhlzubHAbgejE;2A8{$ePy`@9jep4hs~PsbBPo?Ov6L*MzvXShcH^qWEG$#pM#iHsN6>#bZa*9U!TdPYwZkLm8c@i{YQ*?Y7c7yC>0 z)RUTF6%oB<7SP@`uL4hRlXK9OZ?bb>f#fEw=CGEqr{{co)WS~NlAb5%Rx#IgD)LJp znBh2LYHdOqcjM!;g*kIdvD-W~Os4caoi1vO4l*LhRR$K+N0#xk)!ogGPX=saHdMwE z+2pk|Laevjk~t;{*gZ>J#MF*FcXn(pD}BD*J20t80zyhDznko@C9+?(>#RkF@qJCT zyPo;42a^7`J3kfw`_9kX#AraZrs1e?g6V{KhabKuKW^ti0)@le3`E<>z_8V^E_`^MH|Cjj=5}b zX}MeJl?Hu&t7CxD9?3Lm2W>gSn5l7LH*>pGXKymMe@);mXU3jf$F#oHwKL?Im8oUh zy#Dgtj_@``^g_7uJ>A3=IPN4|mrjVV7VC4A@B536Y>P__t!^|@jn&*Eq>%2&3VwI| zk!0R(UMeR|_zNp~mnY`?iJ_#46%S2%@f$q*_!XD-7I8BjN;30YHni>N$FE|lu|4~N zhs#=}vf&-XU3+w+(U>pz|Lpv{IhOSICgt;N#wlSKFlKWFbz1;L;3)&FKkY(3|ML`W0|W8Pd}Hh-B%}bm+n3 zoYO{)cSva`{S0N(a>>ch6R2bylwDst;5uo1nKF)woX=OfY%68j&NojTlN6Nhn6AQF$d zZ=W}_TNz<&9iQ`;a?^_rN=h+193Yrnes5AvfH3_&mV^tX#QME+SQmkg@$xyv*seN=GrUB!9~FJ-ksja7;}N_wS~Q4XH$Dk1HSV44 ziL5NdPFPw#Q7y?Ovq+uD_>O;1^Y~CyKbc5=`Y~xXHp^R^UZU zZ~y3z?3lH1Gz3_2gA3V%tq?&*MZfx!{+5fO*&p%?#=;AxQJD;w8MNiZ|#y4iP@sPA*? zqWoYgk|3-(y5s+zb?TZUD-6`*v^Id z87z+KB4~>pKM@2h_l6-0Gkd9Z*95p&Dn19;?HYXsG&SpZ1=@qK9TYWJonyFPvrYC1 zIbGo*<$aISoN_fB0%{J?p&4H-7B@`}%?YhBw(my|kGYVRtKahVht1x>`4pB2{c*vO zR<$XpaxDlUarH|^UEjvm<}nE;JfUjUkSXnYvinYzLAQT=T#qaNTBQ^&@Au76(-Tf8 ze>f%ZtOHe;)c}o3b1wX^+!rN{)LeB{R_eEY2)GW-%c)x zgD023tp_(}mVTdvMG#}4qhTU$&cOnI9fU;9;M5R}m|oRe*lFM;SBo-{_wuy%CH`S!9_74@36;tTZ$W67cmS;7+GNeNXID=8c^ zm|rfLhOohx$fBRbBCjmF*o5}(H-w&9Hhz9E^o6RaFe4dxc_w6%9g)sAV(0n&5bK1x z;^Ecs6f?7POrdQXPr8L-7@P4{y29k)Gru8RN{MU~p`O#U+qT&zD&OSFtjeY!O@WU> z?0%r?uvn9QCa!KQ?%aG@lU+aWQkRk1CP2m1jv_Qg7HD*kVpwamBA1bze;+aXXq?zJ z%T33`A8FKiaoDOD6h{1X_QkMtrfD+P08m+s&gc8X@X4c~>X=yMIpfiH`67{D&y|jt zhuyX-a;>M0GTHd6a|&965Qxo{CiS^~X4Dn_v!N`k`id4s?@qn=$a*FG-hGcn!tV*} zcYE)-?sih%>vFnFOWP^!Vt#!!+E;IFv&5>;nCn*2)pAdT|0;IJDH*c><8%M5alhD? zJ#n{39t1+8;#PV4rby!>#avOsr{jVO8z2`rROLC&&G>@+wYw^_eo`bb?H?f>Ix#p#pEHtP=l?1dJL&@ zB2^lAx}VdntB{KIfs#`dRoJ{gn_WC-A!g=uTzuH;Fs^!dHR&@=-q~>#p1JPRYr9w4 zcdohcj2nFyIFAY~t5Adk;yc4hxcoUJR&OgpTt z-4BR)du~Ok_NBFJ%TCX>_*RL-9dr{5<8NlIvn=6{QqidBT7R{umLSa@DO6AJPOE2G zSM8bSWQ09E7`#6+F?U{zPQFJ2o$eQWeCa#z7L#m5>Kjdefr+I>f%ffKVC(#b z$ZDpRTBf$tzEW0oLL)RiMEne@eyyM%9wEZvtVNcIA*~a&u56!DYGvb2Yc3xZR1J`C zp=lqSYQ$V(L6W7CihR$HyXqsC_{va~gDx!tE8Vz9KHoF(R)CYf)wcfhX*k=d_od>a zlFj23R^u$HxRh6nv{5hr$2UoB`hydYgKB7 zXUD>BmIh4!T#$-F*?<*PeoKaC;#<2J4 ztAvM1vA7nqVH|btjW56QFj3bHJmV#N=;)(Z^T-;O*JOIsr**r+q1oV zfm4zbu7ry_*XV{d#N2Hq6V(WN?i1H}QW3lOD8nX%rp@WhM?A9!?}Pc@(^Gm;Vs4Gz zO3vpodbWjX7J2!Af%$`|jV2b3?N;;1%bM5Jj~X~nO0x&~JrzvuY!Hk((WANAUOZ6q zEQ;q%jUq56Z5h9P+l*TnAZK9}mS<2G7U5j`=x&;lxyt82P^ENb)!oA(Kk~UdpWfRg zQ&o6kA=40J|8%{-B!@s4Z4PfiOFQAh8@oi^--2xWZhY&Y$R`um#3N0p=ncUt@(HavOlUb62osmr}P`WANIrNGYN!>RturQu3e z1)Jw&ZL14A$E`Zf(G60`J>lH_T7zQmCSvW=KO`4>lR$bd zZ4|CpKz*zmygjAF#g!_FJ4B^5uJ?ds?>(^~`Q#|HFeE~ErL`g+T`!BZ3m?=t*Z&yTSue>uF+n{)VRH|Oxd`RWPi_Wrx& zs5r+1(moT)V(U3Q->`EH=}by+JEF1KV;Q{<$Mah6nxv=yJcC^z@2q+OHP9b?QF1o! z>e+tVj#}A~@SSX(OSqK|N7!4&#OFF>W@3Xgd>llRF-iusle?h zxa)(WsWYS6lH8pyHQ%UzF_=V*E#wDT328PAetlp~l89BE*m-7INuxUG)KXM#;-@J< ziTQ))s`t}b=_?BZy~H|x^Flm@#td z8~-8i7TR(7ZFu(+>ho({GZjOG##EIow#sALA0eFMvptFoc=SlQIILRy9E-&IJ3@=2 zoOXs(HMol_+p%0X&3^v?43rbF^1aAl_V= z51;c8e=x16qn5h=>5cQ0SgiI-0$>ztTrI9xZSA#!9BK*L%A+wzQ86jGYN8b4Bf_FD@=o|vE0~unY`1pn|yI#spgXll5KJ2%v-vMG+Zm2<5|K{Kp z{cjLJo%sib|KFd!y`lZ*1TJXn%!TBlL0ebXcbJz0r4W~(m4ZiTAKB`}j$XU= z`~a4`6Ng=p-cQ9K-MII|^Q+ttp{DD8#^V53Nm*aaE`)0i!*WMe(?t9oWlsr}Bi+Qi zRXn(W3i1BAr<1$otL|}jqe^6{hm#)p9O<2S2de<4f?2pWaCue_8~bysx&s zv{xKPneelBb>$}E(Se;SG|Mnv|By{lq^b{bVw>OA?+t)wOMYlTcwX9z~U4Y@5 zYczxACtcXFxr1o=P%~5o@7b*vP1=O0bm{2>HtruS(y2LiKe1le8**cNHqw|p+FT>J znjbm|D4ICzTHVaI(`ZmKUW~ORdmr0eCN5T{g}Kpm^0aUIJ7z66=BJ<+sbh#C1tyLw zyv8*BSeDOIQ~y_OUlmo?wrsl=Zowf)&;)lD?hqV;ySuwf@F2n6-Q9u)hv4pR2@ry_ zfFMC0WN+>{_uPHrI#xQw@kp31ag7mHdKr-TP0pwrO2AIQROd>L ziou@YbP@+>>0%cN5H(0F%5`uqzJw}Xqt;kG(zZ|8_ESzA&jhZ}D>?Z8Mgi-D*;9u%1+sYnZdx)jh) zH28xFJubR?h@#4gGe(bP7Io;)Qih-6?fObn=Y%dSILYg@v8KrQ7A56Fl6!EO$&d;( z^T`?G*sX5k56pWkc>x`U-X{TvTm%KzL9R-{tRVRgFhiqhaQT2Pg{`A`EfH^W15*$L6zttt0(!c(i!Lqv>yK3Yc2!n_K;VfFkM>)jvaVHV74v8J^Q;OqWqV z-K<8r5oPzr!Ntj~kB)8*G?qi=LAm08cQaX-;0O~ei^qubeDSUM*g!zoWN;f`XW?4)g%3R6;N)eJvOjro~|WonYg zT<&UjR%Wax0tE}pKZDg>4P}=&vhrwok@s)q^e+20esy4LH7$2Vd%*7PM(tiVvc_ea z+Vk;6#%V5AQCzSEl8zsN%;@u=No&hIGz7dNJh~4na1qH5cU}u}ZDEn5f4ahC5FRmV zZ22G#+Q{q2g5=TotZ>Xn^eTIL^nI{k$5;%HmZ2k{-G!}&7N4dye06vRQ^bwR zlKObZhrg>c79(-Zk4mj zh~B9Fj0<~^&iTM0Ki|_^@a^KbblcoJiwT_Qqpj+FTO~3VnP_^$CNH6b_8socTOBE8 zkIRcRyznn4?&$BbIiQL9wyQz99r9^Hx*r*gPt#Ki#ZbIRB5#vGb16WtA^<;a00nf zzjUG5S?m1CpJ3!hh zuVpMfo#6aQ9QZ7II3l^=`fq%l93zSDF)l6nM&@3+R{O=^`Rq|^BwXJMy$|$lqnYNF zJ;JQU8pNjVJC@Z8!NP&(M!*1cyX<fg8^h-z_Yz^57S!J1^y6;Y=i z)a|ijDfh0pRSaM-?2d87FXtNcL}1ZSUSuVnvas=9Lr%lgU=!|aFdo^gtgLOfgUxim%HK*29t6Md7Z}?>D-3R&5-wk2@u_hPn-!+m ztk<$P)EtFvLouq1&tu>{ognYoD4rBfC|0P=g*lUme3z6*(3+1TawI=T4~+ARv8*Et zH&=_aPxUiCAuDi!77Nj^pSMD;v-{fqD#_8<3nf-6V0cL&ok)#o3tnGTI=g{X*Py1S z)k83@^sEYul*>9M2ZMFWBqv5`HGWdkU)uTECzaEQj3GFnX;~!;jWFBeg)n?)y(B*W zK$YE6@{IPPdSUYIU_>KIZwP0y+wJi9(tB)FG3ryc$uJLv`j=O-x;*F6FVeN^dBC*W zF$&W;jIk)8P-vksDw!zS$1}S32rsuU(fQaqJPI|aC2955B{XHdLFij7hlzIrJ2{#R zLCy%rwLGwR*%b+J;N5#yQ{-zSB8GEgoXn*g3fdwH46xH0+DU0d=|m4mZ411+tLXj6 z`O2@#gnCZ8F3RgrZjcgV2$4v}3kXzOaX9eB*VDyKJ8O?M75OQMxyZ;Zrg=bF18g^? zd`65(&T=j{2Pi#QtJ4pPt{LzY2}s2jSRPj2T33U?27cVMlm|C-MH{D*?CUz9KjT%l z-|;H(ztvbzX!$dYedaa-YLo|+Dvj?%`Y9kt{~`cY!prO)FelGU=!dtM@)-Vd$x6~FCLTORb7tdjP zkCh+}0yFGa+rm1rp$hn}dg(J1F$fX6~gxufg9gTAzX zGXt(iC7pN#2AY&A5f3C*x3xEcu?>sKImEEE;7L%B$L)cZlTFRCS1Yf*VzsL+uCzb)!ZzquDgZ5dYfZP=H4hA}L)4_@Fcty(o^}z4 zjUhmv)Y_RP#AP!TYgPFZJ-TJMrGow{8V8)Iw~!$_{br8Ps75&Bw7AYq2%d%}EA=i;Cv#$10WKKPpxTT~A@GhFLKX_NlMOAJ_lOij|HM|IlOD+GS2?JHQ1Bi(RhoF7hzBHKeCHLO}>iMqa53T^P}U zT_M`XQ4-3!vf3dom?(r|xwS!qV&xfqiH@~~%&Uo7PrsGmA)!UnbgvZa+r3hr7T~iV zg?TzL`C=XMR|a|FK6955V&(J0xRM0+=?BQA6{Wf^#}o2JGWs_C9CmY# zPD}f0Gm#J`Y%d$iu9AC^&Suv39TY={-v}qL zJMHjS0FIdcuhTk&qF_etCf70P5y>Ts9cF-Ll8UMrw46-Y(hG7aF>9?CV$I7^e2oGq z8w^}Y)AGG7({cnPo`V>WTp`@PpT5+4xR|UsifxM5yo^v3VH}k_294|_x}0c3q9CE> zechER8d-Bp`Gz(dpK#A~kkb&4X&T){5Z?hCrTbygw zOr};1>RmQBBiqN)-JcChzw)x!(s9Qrssz1coV1AcK-Snu;~K@WO2H2ogP2}NjX>tO zUiqd=^5P-^&1=V{csSd?3R~I#XXC7=)~=4`rZ#l;whq?jfAr4!-I?!iTDn3g{SIUK z9{uQFK^+=GM@jJQQ;Z7_+ywdlu5$fFr28b>o~?tz3>8q6>oON&k>Q7ioEdy%Y1d4q zTx8-K8U>X)eYlNs;RxK=&zRl<;W}54L7kNlaY(OaU{C5r72!AIZLzeijWCyzC6NRt zcMx&t(L$aJWAIe35;I3J_glB^HJRTjE`2z@Dhp#|lK}c^6@qEH-{m%&QNrx3@C!jG!)EHuWr?b`UlmkH-8 zqZ6H~ak9wJT6}WF5qy7i@6P>V-Lg%uqyuuY4yOdJFJMQBi5DCc+5Osib#+%;NwVDBvJfL|BZfvVzw}v$0fx_b#?LdUjs1iCrDo)?$lfNIV7??zU)g%Z|PR>t>v>O zaaCPEID1(Lj@xbtPkYFL_#>*<<=nK;P}q>{eBVWN(6PT}&m@L%Q_&na7{}(lA8VO% z=r*%}L5zhDEYOhK;20?(_P)dPgk^KYFVZWaq^S*Q5$ z?B62_edkSl%G>Lap0Lg~*EX*Wxj@}$`=fqGI(F7ZsA^V;mVcGGVaL@S!W%2?VDlr3 zoJZm&V0Hmn+evK3xkdE_8s-a3Izz99PYZ|Ur&;PHi{DSoo`cN`ctMg=@*(fj6i9YfdZHh1v3ArA)S~v}s2mG95JTq;nH}ES~ zZeL2L6A2Ht!^dic8BsIG<3zHmybCY#qcRqzjVHwSgu;oaFPEJyJqZWmjbzM^?lg}# zYJ+SSx4cxgXXv3&&E^?y=ylJ0>G2D&=b_hO7&C zkSJHtOv>B`?6yNbXYwY^%m*7Z5XUr9)B$d+E&FyQD_o!lGTPtTe>)G4qt=4p`0f&) zaM-#A33l2S4>b@P$MNspxq$o(8os!tr$?^g6Kc<+)boX2;;1!)w*as+tV8`Qtqm+f zOIY(#$Gk#Q+Vf$z9($=`v70jH6XX07>?#B4yqn9L%TQESS9YnxLSAS&Q&8L{I6uXE ziF`p^EBs#nw#0Db8KqCFpMEnf9H79K-x-82=U+@Q2lC@1yhHR<80v?63p>w^H$Z{I zci#DDJqMjCpuF%@vgBo24#Ut~_BK%mY=$0Y-gQMP;yVO>!vW1gL!PDi z;OEQz#Nm+UQKQa6eN5AD{4xnDuQOS^=OsRTqD04vGN+Qo-h`eAtqYiM}C<`SQi{Ln4vaWCA+3YsAl2S+^nxt-bry@s?TJO#R6FD z6hLcM`iWtjjMIb^zc*fg#_G%{#>%VMBoE>+IFEI(>0U=l)F?U#o%JVU&IT|quN9Xj zMoR~F#@iW%w?g|83Ehye*Y~&o+WYR|FtU9|beGJc<)=W>yh~?Lg)ecBYhJDa))W64 zs+@ExsLXxSB{4>sm8`&TEIdt_Mh4N^)Z}~JZOc)L!*pyjWw<6^x$P4aEsm`~n9SE0+&Q|5ID;)NQ7OKM1LG0I7jPjN+;)^%C^G1AFy*8Za9r23>vgEo!>-u+o zH;G;Q7}UczwSH&WwN8uaFVcQ8Ect)8pM~Sr$5xyeO|B&i!+8m>WbhrjsSIgL(2dNP zhVbOmAjEyZ3L6q->G(@0$58^wdq(5*T=FaIsL&wF1)KmDt@)MNix?ipLL{$373dZp ze8HI{C9AlZIR@Ji$cUCg$)VaN*Qya|Q=5U~Peva~mGWCWp3O=+JICz;jY8rRWh3zz zb`PvEbkfhqO?4H2O6ra``th6_w_&h`L9Y4Fak<97@rS_ z1gM)b-sZW&-iqZ#ecXt^9jkk5J_0BGU}&q$&LL^h;trM`NBqGj*3~sXY;m*ZT6$W_ zs;iY~!_!U!jL4$I2`|%o0gcbJw+c*BTQ;ucXEeDug(ce(=Fd`K(HOMx?=zS1Ua92u zz5dj+fT_ZTuq(JECDW)FF&}%}j%2ofctkg~K-;0uAHPpSGcM@aWop3DOH^kM8bfn6u^qWv*rKnvnQ50r4!Q}28JF6zPBqP9@Y7Or} zp}#j+>*xD=ZR4`^(f+vC@N5i>_^=%hqxSk+4E};*l@o)#O5m+ZN=+*bq`fR9Q5cpH z%`J(>N>vpczp8*7=C$)jH=foyX+@CpPXR=#`8zB;TQ?(NX9=w66P+} zjx;EqU}R>agk7wI{R)LpL1dK57O-!{_JzW}ysl>`~q#05~2!*zU zq=-d)FdqCNzbY|J+sN@Fo%ov|8~q1$Mzqt|_@PZ!Hr<8~tR;a*ik(9wlu&t!1^)Y? z`&v4D-gZfyy0Qi9FO9oR5|k_mvU06WE~4R<;OTB(QOx}iq=I#2MGz3ko1y_FP#glG z_JD{85W~A-RYQcrG1>k6d8cJH^Fml0o?k@;48{o*850{i@OJZx{LnCO)2<0MI0V0Dr;X{}gBjUn(M8!XTE#T6FbeDoeW6%04@!e23 zUc{Sp6o$3)fC4ExkNuc^Hl;!!am)|R3Jj)hmN+-{fjj685bv62ZX&((D#a8M@Qm;* z!|<$mj|7qNMG2D!jFM26VuhnSG%ZKP;Gxn>zjdm??!QR#^-!i)FhILk1fT10xZvCQ7di*hr@R=aQG5| zui*2KIbhr-F3&bXcBE6F>sFZwDx%stvoW~wXJn#V@DI5s#@J7#$#4>G_c<00XjOl` zG#JXggQ#s5NU=3{stP8sutV6O4N1KA+WU0DZ+Mf$5umUHqa+a&4&xl$>{&tL6irW^ zyQcGb$tkL}xJ83Y%%6e#o%SdE4X-cSjgT0)8Vf#Czr;PjBp$x++uxCshti(7*UXhr z#MWe8pY*0)yBc$87|SIqdtpBmiH4Ef%i> z;fSL1{SKs7+$-8bn6qnhsifddiJSTHA_c_%x4`@m;gh2h2F%6c zA~a6gngfUFGR7rBvRaLP^&;lK6_@8G` z`4T6TjpU}6S&~$x?~S*e2Bv#9#e>UXKfI)9R-P#3zb84YOZuSNSYWT9sos?jpYC1c zbB{4YNSml=_Jj6T*zJ48cx>cXG_+#83=H)NG!HYln^IYM8Fg|fGhY|WLxjzk(lVGO z2ILCv^DhY9I8v;un%7V=AH8Dv^;gC}S5!{K@w#*&?cEJC;28V7l1UzyF&g8;!MK0Z z=_*Sh30^yXUtbr^Vmo9k-K;S&Zzp!ZtEizhas+aA=7Jj}8wfYjVor^FFxb=Mf%UEx za)uG5WHjD6Q_+RA;-DYLp0cZVGzMo5w!hmhROT$7iH%`;A4sCth~MN8>#hXDlksd& zED1ND!Mt>Z;LO!mC|K0*H){`A9{JSnA7l+&dm?MKF{LJGt!=)L2XErGHM%I9y$2)(l;QkL}oeY99lr=QLhNpo6 zLIIb=P#!yW3gCDv%m=3CIBBgpI^y%kyFOrgGzu}fRdcmH#L2=M830DwAj z1${GTD`Gi)8zbY#;=bMQ{|@?Qe_O-~NrA$^MgRytLiw*o3k~t~TEBx$$ettrKT;wP zr2mo?f&KFkoab*;nF0i4mlCmA)HMniy;0fKV5isIBF7ViwrN8XMu;P6ybTcnmB%c8+3%<-zG`ZyzvyRmg*TZG`}zYFdW@i?n=-d+k~mEbBw+ie`N;meAx>GtnOxge8)gQo$+8R z)|G|-o@*#Q(-*SFUSH9$U+_ya%zusGC2U#=txHE&(^V#{@=a7c81*qb3Y+l{yul(q z;f*&>5m?m2Nah)=>@1-6(+Os;u3 zH;5;3_{TD$|A4oYM@liIryqC}hue4>iAns$gGY_D3L&{&>B#Yx=}z%59n$Q+IYnxM zKGk;XLOX^ruf3SDX83UEd3OYBDow3@uKeZ2TWH&d@2V;r3BwxD0!H}VO6XI?zFPAQ z<_xBX-$fq2%U1hP#eM*F)1P50*`dSP=4p%4swx|Vlv+DictL{oMq~>XLIQ|Q=Mh`! z60Vwu%t-tdTTF5`+xn&n;got(Jl8@j#N+FW!aJ4)h0WKnJBIo_e#hf3=il(idrwl* ze+o=xauSl^!j{_(DBGu>ZxeZD6Z_VKozh+VIjR%EtRE^sncMy);#3orH*3t~?I=50{FGCM4H{e8dl)i|(Y|I0 zf;y&)Su0;>>LU+b!~trZJTl1zGQ3i9?Ei6^H*z z0XXwp06yh6{|ZWv#{4+te;9MODd6w&o6qPw%R>T0?nL$QeN)?X1+i1cVG>;a+ zZEsU^S*7bv`5aE=yBIXu5VYx`+JL-ESi-Xq@U!#W0I3m5gPK5FnmV+u_}wYs+GY&6w5pAWDi8N_6p*x!a-E7?sxsn z^q$0SEt8NstqSwx#o^Tlg$o5iOZpLSy1Er-O4ym+t7Y>uZih_I!+0s&IS1nF4G4Mz zxlJw+MDnYaIcX5?%3Jpsxzkz>Yaau{L(iGN-wV$jo9` zflmG_O{jujF^Yyt3MbMHroy}eK3>uOFC2{vh)v9IPU@yJjtylwOzO-8-*VT#EqOs2 z+VJc}o@C&kHe2mJX8qQT!iYNh9^`g_t*Cgfnwk>IEPPWyEjB64Na3g$)7E&SiQ3EE zhZO$~xoRSzanV~i#ZU{)z$u(8Pa{I<{=No|z2O@4>*})R?iD^638Jy3eU(^0a5ARl z0!sGweA}%kQSn#xe0Xe*@5h(gq?t{lQ>r=SUV;IA674iu4X6iQ<%<$N-$Rgp@mjY{ ztO)4J^w!!&-%)Y9mwfLN@OEOc1ayf_suscSH|)|Ol+N9stWBxA(#1fw4^*fO7G|}n ztj7uj8vMd}C)bM4Ee%zJ(&rZ;vWVe=InwHnBX)yNNO<dv&`4ax+R`QrL=qEsmqNHf%$b1rGhK<+Mp)7LM~|FKWfO`8=-A0>~U(^)Q!}eQgpU zjCAEgQMp=$;0VGXZ~WfUCW4=b!brb5HN(_4Qr1>Hw5<_Q;2uL2X}pb_)~C+njSKrQ zEZTcnmUQ;!6O-%u=a0u7ik08_H82ld`%x?aNEvrBz%m@s{|Y}gi%{MMJqW>dj^y|4M!YY`^O~|Y@n+&tO`iYAluqd)IaMT zfXd_e)+vXi$Y?}RHcGFi@Gd!+@wu`1)!mcB zFnJDl=ibi5bRY)S*OO)u_GIoJGJXB9FGiRA8glmpDt^aCz+6a8q~4P~SX_%VqC9k3+#&jv$~>w=$CL^JrXH@sOfOCU*Em* z*Pi|2ux>4;${paG;@)JMIMZtWoQFJIG44?zl$Uzk`wKu#bSo%cM9QxI)c@km4`n_I zHxIr|oWT^LD1W;{A)^eP7Ccg@cQkPe`W4yV;5#7E$epDF{4>$VywPLh5xo>7MxvI2 zQDQ$86+l8r?_|=Jf(CAU%LQBKbAK%e5WGd+e7#dH64^u6uRK(ga+1~2f{w*~JAZQ| zKQV@7rM=O=fvPJTCs^&59IfKjrELb{=RK(t3zfXGODMbdH&FV z@b;X^o)i!!LQMS64Rwj20rYYPJc7v6;bfl+ZC_2VeDT+Ou;Kz_IU#1QKD8x29T%~^ z6>ikzzJzf5rtY)Z@fpPL6b0)ETP5{Q&D0J+tjRI5{xhDN=t$&i_w3`y+7j zckPk;(OLWzlAs};X8gaBGyhg%KO=9Q2nq0!fbZx$X#@;9$Z-MCE7R*9LsQ=}@1IO^ z^`VfC!VHIDqa}o+BAquFO7nAZX=R(+eI#?98-c=sPg^eOVecs@DH3EhB!Oo=D(ppk z8;M`Kj9@=W;s^=s_nK?datz><9C&^4uHIMrh|wrZ!M z6%$?z#(L-PT3GkeaW?2@XItH`)vRYf!oe=n3(#`Le@kq-%$LT3e%Foqi?2%qvph4^ z5-&epJujSpv~Z(`l>Fml+s<+&6)r43XH2J(Qerq2ans62gD=h;iWM?IBxb~TqQyFY z<|+76X5=Osw3g~6!CqbGJYx?PityXXGJRyIOzr-mJ5-uhB7TbNk~b|iBgZ49A;$Ba z+>0@hN=p+dj+aAKC&`&BP6aHq1~1o+N9ATI9E0JXf7Mxzwxbl!s+CNVXoak> zU)gq}(K1%^;A*)AxlEt4M%)u2+R9d4m~>`TjWGtGOrw(R&7OWwJFAGq6mr;PgvYB%T5b%R=r1oWSxUi*ntWSqWRLXRToic}YN*Itd z=sM{=jS`bcj|PS@V&khoGtwtFDb=e^?-j7z(N)a+SNzVuq-QYn=84MF{qtCn1_*(LAw z29u_Ytj4~@`PL}1Yo~_py!@UTM*?1(4%oOz%NBbzpy1olW5jZHy{eXa$f6Rq>d4X- z_9k?%fR!n$HJ_zoGNZPhB+9w0OGSll{ZC?k%Fn?4&d>avt^Ll=JR20X2jV@sR}NsU zL4y9{Zmcw2hSn&!_rvgg7*33WoUX4&4pg#s*cn`%Y$>`eKj*2s2{V*3w)=?;<`$G! zq`pNA{Lhqz$qORnoh5IUi=l6Bt4gD5e!Z@HU*J!+j^GOv;5f`mqA(GC=gM(H$9n=y zot2|4AJWAaJ{8a%)4ZC3DK4*MHky+rOy(5t^S^bmz#B)l=bfrr=)9n>&E<1XQ>Zqf zZx}Fo%unbg~9-({}RN=XXnR7nV48J zSwy*#j1qZcH#XNRC|2>GSEcFNcQBCraKvcDj2;(Z*f-}V3)N!1sJ zyH{wKS1D0T+HE%I>4idHdn<0V{#1+*Wl}&gZM00pUPrRbj?;*n?H%Mz98dT3ZcPT(4@!2#J3(1xbOS3C`1k|9>V9*zB?12{CI8OPJngLi zkHwV7{ESIYlh=RG&p2Y|4&`W?36x0j?F99n<))9G)7o{6VkhWF_>S)7@BT}EhUzJ& zukc4s|8eWJf$ihQ>!;Hq9iA=)oc}nj_%F_k{HGMnUkw-<;_1zPM|IG7sei6o`M3P> z{FxvVfq;X3TMWIZz#O)Zb3A*9j%f_#gkKlSG+Toi6avN4ErS6MCe(-#O9S#S`VOpK z<;$KaK_++H#z9_Owgg /dev/null || mktemp -d -t 'tmp') +NEW_VERSION_TGZ_PATH="$WORKSPACE/$NEW_VERSION_TGZ" + +cleanup () { + EXIT_CODE=$? + [ -d "$WORKSPACE" ] && rm -rf "$WORKSPACE" + exit $EXIT_CODE +} + +trap cleanup INT TERM EXIT + +echo "Fetching ICU source archive" +curl -sSLfo "$NEW_VERSION_TGZ_PATH" "$NEW_VERSION_TGZ_URL" + +KEYRING="$BASE_DIR/tools/dep_updaters/icu.kbx" +if [ "$1" = "--update-keys" ]; then + KEYS_FILE="$WORKSPACE/KEYS" + GNUPGHOME="$WORKSPACE/gnupg" + mkdir -m 700 "$GNUPGHOME" + echo "Fetching the upstream KEYS file" + curl -sSLfo "$KEYS_FILE" "https://github.com/unicode-org/icu/raw/refs/tags/release-${NEW_VERSION}/KEYS" + GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring "$WORKSPACE/icu.kbx" --batch --import --import-options import-minimal < "$KEYS_FILE" + mv "$WORKSPACE/icu.kbx" "$KEYRING" +fi -GENERATED_CHECKSUM=$( curl -sL "$NEW_VERSION_TGZ_URL" | md5sum | cut -d ' ' -f1) +echo "Verifying PGP signature" +curl -sSLfo "$NEW_VERSION_TGZ_PATH.asc" "$NEW_VERSION_TGZ_URL.asc" +gpgv --keyring "$KEYRING" "$NEW_VERSION_TGZ_PATH.asc" "$NEW_VERSION_TGZ_PATH" -echo "Comparing checksums: deposited '$CHECKSUM' with '$GENERATED_CHECKSUM'" +CHECKSUM=$(shasum -a 256 "$NEW_VERSION_TGZ_PATH" | cut -d ' ' -f1) +echo "sha256: $CHECKSUM" -if [ "$CHECKSUM" != "$GENERATED_CHECKSUM" ]; then - echo "Skipped because checksums do not match." - exit 0 -fi - -./configure --with-intl=full-icu --with-icu-source="$NEW_VERSION_TGZ_URL" +./configure --with-intl=full-icu --with-icu-source="$NEW_VERSION_TGZ_PATH" "$TOOLS_DIR/icu/shrink-icu-src.py" rm -rf "$DEPS_DIR/icu" -perl -i -pe "s|\"url\": .*|\"url\": \"$NEW_VERSION_TGZ_URL\",|" "$TOOLS_DIR/icu/current_ver.dep" - -perl -i -pe "s|\"md5\": .*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep" +URL="$NEW_VERSION_TGZ_URL" SHA256="$CHECKSUM" "$NODE" -e ' + const { URL: url, SHA256: sha256 } = process.env; + console.log(JSON.stringify([{ url, sha256 }], null, 2)); +' > "$TOOLS_DIR/icu/current_ver.dep" rm -rf out "$DEPS_DIR/icu" "$DEPS_DIR/icu4c*" diff --git a/tools/icu/current_ver.dep b/tools/icu/current_ver.dep index 3d923fec865a..577b585578ee 100644 --- a/tools/icu/current_ver.dep +++ b/tools/icu/current_ver.dep @@ -1,6 +1,6 @@ [ { "url": "https://github.com/unicode-org/icu/releases/download/release-78.3/icu4c-78.3-sources.tgz", - "md5": "a7b736b570ef0e180c96a31715a00c78" + "sha256": "3a2e7a47604ba702f345878308e6fefeca612ee895cf4a5f222e7955fabfe0c0" } ] From cd8b5fa5b02ea2a8a01fdcc2c68070bf363d754c Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Sat, 10 Oct 2026 05:29:37 -0300 Subject: [PATCH 54/98] test: tolerate async block count updates in test-fs-stat-bigint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66576 Refs: https://github.com/nodejs/node/issues/57797 Refs: https://github.com/nodejs/reliability/issues/1657 Reviewed-By: Luigi Pinca Reviewed-By: Ulises Gascón --- test/parallel/test-fs-stat-bigint.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/parallel/test-fs-stat-bigint.js b/test/parallel/test-fs-stat-bigint.js index ae8d3857dec8..e70eff046b9f 100644 --- a/test/parallel/test-fs-stat-bigint.js +++ b/test/parallel/test-fs-stat-bigint.js @@ -78,6 +78,12 @@ function verifyStats(bigintStats, numStats, allowableDelta) { `Number version ${key} = ${msFromNum}, ` + `BigInt version ${nsKey} = ${nsFromBigInt}n` + ` = ${msFromBigIntNs}ms, Allowable delta = ${allowableDelta}`); + } else if (key === 'blocks') { + // Some file systems (e.g. ZFS on SmartOS) update the block count of a + // freshly written file asynchronously, so it can change between the + // two stat() calls. + assert.strictEqual(typeof bigintStats[key], 'bigint'); + assert(bigintStats[key] >= 0n); } else if (Number.isSafeInteger(val)) { assert.strictEqual( bigintStats[key], BigInt(val), From 1722ced4b42aaa47f086dbad871e94e3387c167e Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Sat, 10 Oct 2026 05:29:51 -0300 Subject: [PATCH 55/98] lib: add --allow-fs-vfs to permission available flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without it, --allow-fs-vfs was accepted without --permission and was not propagated to child processes like the other --allow-* flags. Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66554 Reviewed-By: Matteo Collina Reviewed-By: Ulises Gascón --- lib/internal/process/permission.js | 1 + .../test-permission-allow-fs-vfs-flags.js | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 test/parallel/test-permission-allow-fs-vfs-flags.js diff --git a/lib/internal/process/permission.js b/lib/internal/process/permission.js index acfbab036bbb..c3b81a79c665 100644 --- a/lib/internal/process/permission.js +++ b/lib/internal/process/permission.js @@ -74,6 +74,7 @@ module.exports = ObjectFreeze({ const flags = [ '--allow-fs-read', '--allow-fs-write', + '--allow-fs-vfs', '--allow-addons', '--allow-child-process', '--allow-net', diff --git a/test/parallel/test-permission-allow-fs-vfs-flags.js b/test/parallel/test-permission-allow-fs-vfs-flags.js new file mode 100644 index 000000000000..68fcac76bd73 --- /dev/null +++ b/test/parallel/test-permission-allow-fs-vfs-flags.js @@ -0,0 +1,42 @@ +'use strict'; + +const common = require('../common'); +const { isMainThread } = require('worker_threads'); + +if (!isMainThread) { + common.skip('This test only works on a main thread'); +} + +const { spawnSyncAndAssert, spawnSyncAndExit } = require('../common/child_process'); + +// --allow-fs-vfs requires the permission model. +spawnSyncAndExit( + process.execPath, + ['--allow-fs-vfs', '-e', ''], + { + status: 1, + signal: null, + stderr: /--permission is required/, + }, +); + +if (process.config.variables.node_without_node_options) { + common.skip('missing NODE_OPTIONS support'); +} + +// A child process inherits --allow-fs-vfs along with --permission. +const child = ` + const { spawnSync } = require('child_process'); + const { stdout } = spawnSync(process.execPath, [ + '-p', 'process.env.NODE_OPTIONS', + ], { encoding: 'utf8' }); + process.stdout.write(stdout); +`; + +spawnSyncAndAssert( + process.execPath, + ['--permission', '--allow-fs-vfs', '--allow-child-process', '-e', child], + { + stdout: /(^|\s)--allow-fs-vfs(\s|$)/, + }, +); From 1ebd6d3677056e74bf055fa0c924ba23eef235a6 Mon Sep 17 00:00:00 2001 From: Paolo Insogna Date: Sat, 10 Oct 2026 11:25:32 +0200 Subject: [PATCH 56/98] doc,lib,src: throw on missing WebAssembly or RX support Signed-off-by: Paolo Insogna PR-URL: https://github.com/nodejs/node/pull/66533 Reviewed-By: Antoine du Hamel Reviewed-By: Matteo Collina Reviewed-By: Robert Nagy Reviewed-By: Anna Henningsen --- BUILDING.md | 6 ++ doc/api/cli.md | 12 ++++ doc/api/errors.md | 21 ++++++ doc/api/esm.md | 5 ++ doc/api/ffi.md | 21 +++++- doc/api/typescript.md | 5 ++ doc/node.1 | 8 +++ lib/internal/modules/esm/translators.js | 4 ++ src/ffi/fast.cc | 42 ++++++------ src/ffi/fast.h | 7 +- src/ffi/jit_memory.cc | 13 +--- src/ffi/jit_memory.h | 8 ++- src/node_errors.h | 1 + src/node_ffi.cc | 21 +++++- test/es-module/test-esm-wasm-jitless.mjs | 22 +++++++ test/ffi/ffi.status | 1 + test/ffi/fixture_library/binding.gyp | 3 + .../fixture_library/deny_executable_memory.c | 40 ++++++++++++ test/ffi/test-ffi-jitless.js | 50 ++++++++++++++ test/ffi/test-ffi-no-rx-memory.js | 65 +++++++++++++++++++ 20 files changed, 313 insertions(+), 42 deletions(-) create mode 100644 test/es-module/test-esm-wasm-jitless.mjs create mode 100644 test/ffi/fixture_library/deny_executable_memory.c create mode 100644 test/ffi/test-ffi-jitless.js create mode 100644 test/ffi/test-ffi-no-rx-memory.js diff --git a/BUILDING.md b/BUILDING.md index bbac8e570b00..1dbc29fe923e 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -98,6 +98,12 @@ and libc version. The table below lists the support tier for each supported combination. A list of [supported compile toolchains](#supported-toolchains) is also supplied for tier 1 platforms. +Some built-in functionality requires WebAssembly or runtime allocation of +executable memory. When a capability is unavailable, Node.js does not guarantee +an alternative implementation of the affected feature. See the +[`--jitless` documentation](doc/api/cli.md#--jitless) and +[FFI documentation](doc/api/ffi.md) for the corresponding limitations and errors. + **For production applications, run Node.js on supported platforms only (Tier 1 or 2).** Node.js does not support a platform version if a vendor has expired support diff --git a/doc/api/cli.md b/doc/api/cli.md index 677eb4dc4a37..bcd6ad579a36 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -2128,6 +2128,16 @@ Disable [runtime allocation of executable memory][jitless]. This may be required on some platforms for security reasons. It can also reduce attack surface on other platforms, but the performance impact may be severe. +When WebAssembly is unavailable in this mode, TypeScript parsing and WebAssembly +module imports throw [`ERR_WEBASSEMBLY_NOT_SUPPORTED`][]. Node.js does not +guarantee alternative implementations of features that require WebAssembly or +runtime allocation of executable memory when those capabilities are unavailable. + +This flag controls V8's runtime code generation. It does not impose an +operating-system restriction on executable memory allocated by native addons or +[`node:ffi`][]. FFI operations that require unavailable executable memory can +throw [`ERR_RX_MEMORY_NOT_SUPPORTED`][]. + ### `--localstorage-file=file` + +A feature requiring runtime allocation of executable memory was used in an +environment where that capability is unavailable. This includes creating an +otherwise eligible FFI Fast API function, or allocating an FFI callback when +libffi cannot provide a closure and executable memory is unavailable. + +Node.js does not guarantee a non-generated-code fallback for these operations. +This error can be caught without terminating the process. + ### `ERR_SCRIPT_EXECUTION_INTERRUPTED` @@ -3632,6 +3648,11 @@ A feature requiring WebAssembly was used, but WebAssembly is not supported or has been disabled in the current environment (for example, when running with `--jitless`). +TypeScript parsing and WebAssembly module imports, including source phase +imports, report this error when WebAssembly is unavailable. Node.js does not +provide an alternative implementation for these operations. This error can be +caught without terminating the process. + ### `ERR_WEBASSEMBLY_RESPONSE` diff --git a/doc/api/esm.md b/doc/api/esm.md index 7bee67774583..ef15a51dcfe5 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -745,6 +745,11 @@ imports is supported. Both of these integrations are in line with the [ES Module Integration Proposal for WebAssembly][]. +When WebAssembly is unavailable in the current environment, importing a Wasm +module throws [`ERR_WEBASSEMBLY_NOT_SUPPORTED`](errors.md#err_webassembly_not_supported). +This also applies to source phase imports. Node.js does not provide an +alternative implementation when WebAssembly is unavailable. + ### Wasm Source Phase Imports > Stability: 1.2 - Release candidate diff --git a/doc/api/ffi.md b/doc/api/ffi.md index ee56074a682c..d3c01f6e0529 100644 --- a/doc/api/ffi.md +++ b/doc/api/ffi.md @@ -45,6 +45,13 @@ The following targets are not supported by bundled libffi: When using the [Permission Model][], FFI APIs are restricted unless the [`--allow-ffi`][] flag is provided. +Some FFI operations require runtime allocation of executable memory. If an +otherwise eligible Fast API function cannot be created because executable memory +is unavailable, creation throws +[`ERR_RX_MEMORY_NOT_SUPPORTED`](errors.md#err_rx_memory_not_supported) rather than +selecting another call path. Callback allocation can also report this error. +Loading the module or a library does not itself require generated trampolines. + ## Overview The `node:ffi` module exposes two groups of APIs: @@ -465,6 +472,11 @@ Returns an object containing all previously resolved symbol addresses. Creates a native callback pointer backed by a JavaScript function. +If libffi cannot allocate a closure and executable memory is unavailable, this +method throws [`ERR_RX_MEMORY_NOT_SUPPORTED`](errors.md#err_rx_memory_not_supported). +Other closure allocation failures throw `ERR_FFI_CALL_FAILED`. Platform-specific +libffi implementations may provide callbacks without dynamically generated code. + When `signature` is omitted, the callback uses a default `void ()` signature. The return value is the callback pointer address as a `bigint`. It can be @@ -640,7 +652,8 @@ met: PPC64 always use another call path. * The process can allocate executable memory. Node.js checks once per process whether it can allocate memory and mark it executable. If that check fails, - this path is disabled for the entire process. + creating an otherwise eligible function throws + [`ERR_RX_MEMORY_NOT_SUPPORTED`](errors.md#err_rx_memory_not_supported). * Neither the return type nor any argument type is `function`. * The signature has at most 8 arguments, and every argument fits in the argument registers available to the trampoline on the current platform. @@ -650,8 +663,10 @@ The register limits are platform-specific. Integer and pointer-like arguments share one set of registers, and floating-point arguments share another. The limits for each architecture are listed in [Type names][]. -A signature that fails any of these checks is not an error. The function is -created on the next call path that supports it. +A signature that is unsupported by the Fast API types, argument limits, or +platform is not an error. The function is created on the next call path that +supports it. Missing executable-memory support is checked only after signature +and platform eligibility and does not select another call path. ### Shared buffer call path diff --git a/doc/api/typescript.md b/doc/api/typescript.md index 3959fd58b56c..701280375ba5 100644 --- a/doc/api/typescript.md +++ b/doc/api/typescript.md @@ -82,6 +82,11 @@ Node.js will replace TypeScript syntax with whitespace, and no type checking is performed. To disable this feature, use the flag [`--no-strip-types`][]. +The built-in TypeScript parser requires WebAssembly. If WebAssembly is +unavailable, parsing throws +[`ERR_WEBASSEMBLY_NOT_SUPPORTED`](errors.md#err_webassembly_not_supported). +Node.js does not provide a non-WebAssembly parser as a fallback. + Node.js ignores `tsconfig.json` files and therefore features that depend on settings within `tsconfig.json`, such as paths or converting newer JavaScript syntax to older standards, are diff --git a/doc/node.1 b/doc/node.1 index 4cadf11050ad..38942173e6ec 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -1133,6 +1133,14 @@ Opens the REPL even if stdin does not appear to be a terminal. Disable runtime allocation of executable memory. This may be required on some platforms for security reasons. It can also reduce attack surface on other platforms, but the performance impact may be severe. +When WebAssembly is unavailable in this mode, TypeScript parsing and WebAssembly +module imports throw \fBERR_WEBASSEMBLY_NOT_SUPPORTED\fR. Node.js does not +guarantee alternative implementations of features that require WebAssembly or +runtime allocation of executable memory when those capabilities are unavailable. +This flag controls V8's runtime code generation. It does not impose an +operating-system restriction on executable memory allocated by native addons or +\fBnode:ffi\fR. FFI operations that require unavailable executable memory can +throw \fBERR_RX_MEMORY_NOT_SUPPORTED\fR. . .It Fl -localstorage-file Ns = Ns Ar file The file used to store \fBlocalStorage\fR data. If the file does not exist, it is diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index 84ff29f08039..8b90849143d8 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -54,6 +54,7 @@ const { emitExperimentalWarning, kEmptyObject, setOwnProperty, isWindows } = req const { ERR_INVALID_RETURN_PROPERTY_VALUE, ERR_UNKNOWN_BUILTIN_MODULE, + ERR_WEBASSEMBLY_NOT_SUPPORTED, } = require('internal/errors').codes; const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache'); const moduleWrap = internalBinding('module_wrap'); @@ -558,6 +559,9 @@ translators.set('wasm', function(url, translateContext) { const { source } = translateContext; // WebAssembly global is not available during snapshot building, so we need to get it lazily. const { WebAssembly } = globalThis; + if (WebAssembly === undefined) { + throw new ERR_WEBASSEMBLY_NOT_SUPPORTED('WebAssembly module imports'); + } assertBufferSource(source, false, 'load'); debug(`Translating WASMModule ${url}`, translateContext); diff --git a/src/ffi/fast.cc b/src/ffi/fast.cc index 24809e744a20..4f6606d50251 100644 --- a/src/ffi/fast.cc +++ b/src/ffi/fast.cc @@ -330,37 +330,29 @@ bool IsFastLibraryGuardSupported() { #endif } -std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn, - const bool* closed, - v8::Isolate* isolate) { - // Bail early if executable memory allocation doesn't work on this process - // (missing MAP_JIT entitlement, hardened runtime, SELinux execmem, etc.). - // The self-test runs once and caches the result. - if (!IsJitMemorySupported()) { - return nullptr; - } - +v8::Maybe> CreateFastFFIMetadata( + const FFIFunction& fn, const bool* closed, v8::Isolate* isolate) { // Check signature-level eligibility (type checks, register caps, platform - // support). Returning nullptr here lets the caller fall back to SharedBuffer + // support). Null metadata lets the caller fall back to SharedBuffer // or the generic libffi path. const char* eligibility_reason; if (!IsFastCallEligible(fn, &eligibility_reason)) { - return nullptr; + return v8::Just(std::unique_ptr()); } - // Reject unsupported result types first. Returning nullptr means the caller + // Reject unsupported result types first. Null metadata means the caller // can still fall back to SharedBuffer or the generic libffi path. FastFFIType result; if (!FastScalarTypeFromName(fn.return_type_name, &result)) { - return nullptr; + return v8::Just(std::unique_ptr()); } if (fn.args.size() != fn.arg_type_names.size()) { - return nullptr; + return v8::Just(std::unique_ptr()); } // Keep the initial Fast API implementation bounded to signatures V8 and the // platform trampolines can describe without stack argument support. if (fn.arg_type_names.size() > 8) { - return nullptr; + return v8::Just(std::unique_ptr()); } std::vector args; @@ -373,10 +365,10 @@ std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn, for (const std::string& name : fn.arg_type_names) { FastFFIType type; if (!FastArgTypeFromName(name, &type)) { - return nullptr; + return v8::Just(std::unique_ptr()); } if (type == FastFFIType::kVoid) { - return nullptr; + return v8::Just(std::unique_ptr()); } needs_bigint = needs_bigint || NeedsBigIntRepresentation(type); needs_callback_options = @@ -384,6 +376,16 @@ std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn, args.push_back(type); } + // Check RX memory only after signature and platform eligibility, so ordinary + // unsupported signatures can still use the non-generated invocation paths. + if (!IsJitMemorySupported()) { + THROW_ERR_RX_MEMORY_NOT_SUPPORTED( + isolate, + "Executable memory is not supported in this environment, " + "but is required for FFI Fast API calls"); + return v8::Nothing>(); + } + auto metadata = std::make_unique(); // The platform-specific trampoline is the executable entrypoint V8 calls. // If the platform rejects the signature, the whole fast metadata object is @@ -391,7 +393,7 @@ std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn, FastFFITrampolineConfig config{fn.ptr, closed, isolate}; if (!node_ffi_create_fast_trampoline( config, args.data(), args.size(), result, &metadata->trampoline)) { - return nullptr; + return v8::Just(std::unique_ptr()); } metadata->arg_info.reserve(args.size() + 1); @@ -418,7 +420,7 @@ std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn, metadata->c_function = v8::CFunction(metadata->trampoline.code, metadata->c_function_info.get()); metadata->guards_library = guards_library; - return metadata; + return v8::Just(std::move(metadata)); } } // namespace node::ffi diff --git a/src/ffi/fast.h b/src/ffi/fast.h index 7aced6a7cc54..d7c8cbadb203 100644 --- a/src/ffi/fast.h +++ b/src/ffi/fast.h @@ -73,9 +73,10 @@ std::shared_ptr CloneWithRawPointerArgNames( const std::shared_ptr& fn); std::shared_ptr CloneWithFastBufferArgNames( const std::shared_ptr& fn); -std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn, - const bool* closed, - v8::Isolate* isolate); +// A null metadata value allows signature/platform fallback. Nothing means an +// exception was thrown because an eligible signature requires RX memory. +v8::Maybe> CreateFastFFIMetadata( + const FFIFunction& fn, const bool* closed, v8::Isolate* isolate); } // namespace node::ffi diff --git a/src/ffi/jit_memory.cc b/src/ffi/jit_memory.cc index 023b55757691..e6dea459c62d 100644 --- a/src/ffi/jit_memory.cc +++ b/src/ffi/jit_memory.cc @@ -23,13 +23,6 @@ namespace node::ffi { namespace { bool SelfTest() { -#if !defined(__aarch64__) && !defined(_M_ARM64) && !defined(__x86_64__) && \ - !defined(_M_X64) && !defined(__powerpc64__) && !defined(__ppc64__) && \ - !defined(__PPC64__) && !defined(__loongarch64) && \ - !(defined(__riscv) && __riscv_xlen == 64) && !defined(__s390x__) - // No stub emitter for this platform; nothing to test. - return false; -#else #if defined(__aarch64__) || defined(_M_ARM64) // AArch64 BR LR: 0xD65F03C0 constexpr uint32_t kInstruction = 0xD65F03C0; @@ -51,7 +44,8 @@ bool SelfTest() { constexpr uint16_t kInstruction = 0x07fe; constexpr size_t kInstructionSize = sizeof(uint16_t); #else - // x86_64 RET: 0xC3 + // The probe is never executed, so this byte also works on platforms without + // a Fast API stub emitter. On x86_64 it represents RET. constexpr uint8_t kInstruction = 0xC3; constexpr size_t kInstructionSize = sizeof(uint8_t); #endif @@ -97,7 +91,7 @@ bool SelfTest() { defined(__ppc64__) || defined(__PPC64__) || defined(__loongarch64) || \ (defined(__riscv) && __riscv_xlen == 64) || defined(__s390x__) std::memcpy(code, &kInstruction, kInstructionSize); -#elif defined(__x86_64__) +#else code[0] = kInstruction; #endif @@ -129,7 +123,6 @@ bool SelfTest() { munmap(page, page_size); return ok; #endif -#endif } } // namespace diff --git a/src/ffi/jit_memory.h b/src/ffi/jit_memory.h index 259c1c46cc72..f5ee3d4315e2 100644 --- a/src/ffi/jit_memory.h +++ b/src/ffi/jit_memory.h @@ -4,9 +4,10 @@ namespace node::ffi { -// Returns true if executable memory allocation (mmap + mprotect to RX) works -// on this process. Runs a one-time self-test that allocates a tiny stub, -// writes a ret-style instruction, and transitions it to RX. The page is not +// Returns true if executable memory allocation (mmap + mprotect to RX, or the +// corresponding Windows APIs) works in this process. Runs a one-time self-test +// that allocates a tiny stub, +// writes probe bytes, and transitions it to RX. The page is not // executed: a successful RX transition is the support signal, and executing a // freshly generated probe could crash the process on systems that block it. // @@ -19,6 +20,7 @@ namespace node::ffi { // The self-test runs exactly once (std::call_once) and the result is cached // process-wide. Subsequent calls return the cached value without re-running // the test. +// This query does not depend on Fast API stub-emitter support or --jitless. bool IsJitMemorySupported(); } // namespace node::ffi diff --git a/src/node_errors.h b/src/node_errors.h index 6e29fe8584ee..8fcea26f6b67 100644 --- a/src/node_errors.h +++ b/src/node_errors.h @@ -125,6 +125,7 @@ void OOMErrorHandler(const char* location, const v8::OOMDetails& details); V(ERR_OPERATION_FAILED, TypeError) \ V(ERR_OPTIONS_BEFORE_BOOTSTRAPPING, Error) \ V(ERR_OUT_OF_RANGE, RangeError) \ + V(ERR_RX_MEMORY_NOT_SUPPORTED, Error) \ V(ERR_SCRIPT_EXECUTION_INTERRUPTED, Error) \ V(ERR_SCRIPT_EXECUTION_TIMEOUT, Error) \ V(ERR_SOURCE_PHASE_NOT_DEFINED, SyntaxError) \ diff --git a/src/node_ffi.cc b/src/node_ffi.cc index 4ef22a80a428..0cce543e4695 100644 --- a/src/node_ffi.cc +++ b/src/node_ffi.cc @@ -9,6 +9,7 @@ #include "env-inl.h" #include "ffi/data.h" #include "ffi/fast.h" +#include "ffi/jit_memory.h" #include "ffi/types.h" #include "node_binding.h" #include "node_errors.h" @@ -290,7 +291,10 @@ MaybeLocal DynamicLibrary::CreateFunction( // signature, fall back to SharedBuffer for supported scalar shapes, then to // the generic libffi invoker. std::shared_ptr fast_fn = CloneWithRawPointerArgNames(fn); - info->fast_metadata = CreateFastFFIMetadata(*fast_fn, &fn->closed, isolate); + if (!CreateFastFFIMetadata(*fast_fn, &fn->closed, isolate) + .MoveTo(&info->fast_metadata)) { + return {}; + } bool use_fast_api = info->fast_metadata != nullptr; bool use_sb = !use_fast_api && IsSBEligibleSignature(*fn); bool has_ptr_args = use_sb && SignatureHasPointerArgs(*fn); @@ -456,8 +460,10 @@ MaybeLocal DynamicLibrary::CreateFunction( // argument is Buffer/ArrayBuffer-backed memory. std::shared_ptr fast_buffer_fn = CloneWithFastBufferArgNames(fn); - info->fast_buffer_metadata = - CreateFastFFIMetadata(*fast_buffer_fn, &fn->closed, isolate); + if (!CreateFastFFIMetadata(*fast_buffer_fn, &fn->closed, isolate) + .MoveTo(&info->fast_buffer_metadata)) { + return {}; + } if (info->fast_buffer_metadata != nullptr) { // Store the secondary invoker on the primary raw function under a hidden // Symbol. Keeping it separate avoids overloading SharedBuffer slow-path @@ -1108,6 +1114,15 @@ void DynamicLibrary::RegisterCallback(const FunctionCallbackInfo& args) { ffi_closure_alloc(sizeof(ffi_closure), &callback->ptr)); if (callback->closure == nullptr) { + // libffi may use platform-specific static trampolines, so only diagnose + // missing RX memory after its own closure allocation has failed. + if (!IsJitMemorySupported()) { + THROW_ERR_RX_MEMORY_NOT_SUPPORTED( + env, + "Executable memory is not supported in this environment, " + "but is required for FFI callbacks"); + return; + } THROW_ERR_FFI_CALL_FAILED(env, "ffi_closure_alloc failed"); return; } diff --git a/test/es-module/test-esm-wasm-jitless.mjs b/test/es-module/test-esm-wasm-jitless.mjs new file mode 100644 index 000000000000..127662845807 --- /dev/null +++ b/test/es-module/test-esm-wasm-jitless.mjs @@ -0,0 +1,22 @@ +// Flags: --jitless +import '../common/index.mjs'; +import assert from 'node:assert/strict'; +import * as fixtures from '../common/fixtures.mjs'; + +const url = fixtures.fileURL('es-modules/simple.wasm').href; +const error = { + code: 'ERR_WEBASSEMBLY_NOT_SUPPORTED', + message: 'WebAssembly is not supported in this environment, but is required for WebAssembly module imports', +}; + +await assert.rejects(import(url), error); +await assert.rejects(import.source(url), error); +for (const statement of [ + `import * as mod from ${JSON.stringify(url)};`, + `import source mod from ${JSON.stringify(url)};`, +]) { + await assert.rejects(import(`data:text/javascript,${encodeURIComponent(statement)}`), error); +} + +// Catching the missing capability must leave ordinary module imports usable. +assert.strictEqual((await import('data:text/javascript,export default 42')).default, 42); diff --git a/test/ffi/ffi.status b/test/ffi/ffi.status index 0cd3513c3cb1..b2548cabf794 100644 --- a/test/ffi/ffi.status +++ b/test/ffi/ffi.status @@ -5,5 +5,6 @@ prefix ffi [$system==solaris] # Also applies to SmartOS # Bundled libffi callbacks crash on SmartOS. test-ffi-callback*: SKIP +test-ffi-jitless: SKIP test-ffi-shared-buffer: SKIP test-ffi-weakref-calls: SKIP diff --git a/test/ffi/fixture_library/binding.gyp b/test/ffi/fixture_library/binding.gyp index 90b1ede01662..d66f2a609f03 100644 --- a/test/ffi/fixture_library/binding.gyp +++ b/test/ffi/fixture_library/binding.gyp @@ -5,6 +5,9 @@ 'sources': ['ffi_test_library.c'], 'type': 'shared_library', 'conditions': [ + ['OS == "linux"', { + 'sources': ['deny_executable_memory.c'], + }], ['OS in "aix os400"', { 'product_extension': 'so', 'ldflags': [ '-Wl,-G' ], diff --git a/test/ffi/fixture_library/deny_executable_memory.c b/test/ffi/fixture_library/deny_executable_memory.c new file mode 100644 index 000000000000..c3fffb883a94 --- /dev/null +++ b/test/ffi/fixture_library/deny_executable_memory.c @@ -0,0 +1,40 @@ +#include +#include +#include +#include +#include +#include +#include + +// The ignored function pointer keeps this fixture off the Fast API path, so it +// can install the restriction before Node's first executable-memory probe. +int deny_executable_memory(void (*unused)(void)) { + (void)unused; +#if defined(__NR_mmap) && defined(__NR_mprotect) + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mprotect, 2, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mmap, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, args[2])), + BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, PROT_EXEC, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EACCES), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { + .len = sizeof(filter) / sizeof(filter[0]), + .filter = filter, + }; + + // SAFETY: Only the test thread is restricted, and the filter returns EACCES + // for executable mappings rather than terminating it. The test uses jitless. + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 || + prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program) != 0) { + return errno; + } + return 0; +#else + return ENOSYS; +#endif +} diff --git a/test/ffi/test-ffi-jitless.js b/test/ffi/test-ffi-jitless.js new file mode 100644 index 000000000000..0877a49adaab --- /dev/null +++ b/test/ffi/test-ffi-jitless.js @@ -0,0 +1,50 @@ +// Flags: --jitless +'use strict'; + +const common = require('../common'); +common.skipIfFFIMissing(); +const assert = require('node:assert'); +const ffi = require('node:ffi'); +const { libraryPath, fixtureSymbols } = require('./ffi-test-common'); + +const lib = new ffi.DynamicLibrary(libraryPath); +try { + // A function parameter excludes Fast API, independently of RX support. + const fallback = lib.getFunction('identity_i64_fallback', { + arguments: ['pointer', 'i64', 'function'], + return: 'i64', + }); + // SAFETY: The fixture ignores both null pointers and returns the scalar value. + assert.strictEqual(fallback(null, 42n, null), 42n); + + let add; + try { + add = lib.getFunction('add_i32', fixtureSymbols.add_i32); + } catch (err) { + assert.strictEqual(err.code, 'ERR_RX_MEMORY_NOT_SUPPORTED'); + } + if (add !== undefined) { + // SAFETY: The fixture signature matches two int32 arguments and its result. + assert.strictEqual(add(20, 22), 42); + } + + let callback; + try { + callback = lib.registerCallback({ arguments: ['i32'], return: 'i32' }, (value) => value * 2); + } catch (err) { + assert.strictEqual(err.code, 'ERR_RX_MEMORY_NOT_SUPPORTED'); + } + if (callback !== undefined) { + try { + const invoke = lib.getFunction('call_int_callback', { + arguments: ['function', 'i32'], return: 'i32', + }); + // SAFETY: The callback is live, runs on this thread, and matches the ABI. + assert.strictEqual(invoke(callback, 21), 42); + } finally { + lib.unregisterCallback(callback); + } + } +} finally { + lib.close(); +} diff --git a/test/ffi/test-ffi-no-rx-memory.js b/test/ffi/test-ffi-no-rx-memory.js new file mode 100644 index 000000000000..f2b5f960932d --- /dev/null +++ b/test/ffi/test-ffi-no-rx-memory.js @@ -0,0 +1,65 @@ +// Flags: --jitless +'use strict'; + +const common = require('../common'); +common.skipIfFFIMissing(); +if (!common.isLinux) { + common.skip('This test requires Linux seccomp'); +} +if (require('node:os').endianness() !== 'LE') { + common.skip('The seccomp fixture reads little-endian syscall arguments'); +} +if (!['x64', 'arm64', 'ppc64', 'riscv64', 'loong64', 's390x'].includes(process.arch)) { + common.skip('No Fast API stub emitter for this architecture'); +} +const assert = require('node:assert'); +const ffi = require('node:ffi'); +const { libraryPath, fixtureSymbols } = require('./ffi-test-common'); + +const lib = new ffi.DynamicLibrary(libraryPath); +try { + const deny = lib.getFunction('deny_executable_memory', { + arguments: ['function'], + return: 'i32', + }); + // SAFETY: The fixture ignores the null function pointer and only restricts + // executable mappings in this jitless test thread. + const result = deny(null); + if (result !== 0) { + common.skip(`Cannot install seccomp filter: ${result}`); + } + + // Repeated rejection must not cache a callable or apply a different signature. + for (let i = 0; i < 2; i++) { + assert.throws(() => lib.getFunction('add_i32', fixtureSymbols.add_i32), { + code: 'ERR_RX_MEMORY_NOT_SUPPORTED', + message: 'Executable memory is not supported in this environment, but is required for FFI Fast API calls', + }); + } + assert.throws(() => lib.getFunction('identity_pointer', { + arguments: ['pointer'], return: 'pointer', + }), { code: 'ERR_RX_MEMORY_NOT_SUPPORTED' }); + + const fallback = lib.getFunction('identity_i64_fallback', { + arguments: ['pointer', 'i64', 'function'], + return: 'i64', + }); + // SAFETY: The fixture ignores both null pointers and returns the scalar value. + assert.strictEqual(fallback(null, 42n, null), 42n); + + // Static libffi trampolines may still work. Dynamic closure failures must + // report the missing capability instead of an opaque allocation error. + let callback; + try { + callback = lib.registerCallback(() => {}); + } catch (err) { + assert.strictEqual(err.code, 'ERR_RX_MEMORY_NOT_SUPPORTED'); + assert.strictEqual(err.message, + 'Executable memory is not supported in this environment, but is required for FFI callbacks'); + } + if (callback !== undefined) { + lib.unregisterCallback(callback); + } +} finally { + lib.close(); +} From 1aa10d80e20fc8ce6e703856536aeb486c890662 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Thu, 1 Oct 2026 17:05:08 +0200 Subject: [PATCH 57/98] process: mark process.finalization as release candidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matteo Collina PR-URL: https://github.com/nodejs/node/pull/66436 Reviewed-By: Robert Nagy Reviewed-By: Vinícius Lourenço Claro Cardoso Reviewed-By: James M Snell --- doc/api/process.md | 18 +++++++++++++++--- lib/internal/process/finalization.js | 4 ---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/doc/api/process.md b/doc/api/process.md index af6875dfa6d5..06df0368f215 100644 --- a/doc/api/process.md +++ b/doc/api/process.md @@ -2061,9 +2061,13 @@ Since it's not possible to build Node.js without libuv, this value is always `tr -> Stability: 1.1 - Active Development +> Stability: 1.2 - Release candidate * `ref` {Object | Function} The reference to the resource that is being tracked. * `callback` {Function} The callback function to be called when the resource @@ -2170,9 +2174,13 @@ as it is not guaranteed that the callback will be called under all circumstances -> Stability: 1.1 - Active Development +> Stability: 1.2 - Release candidate * `ref` {Object | Function} The reference to the resource that is being tracked. @@ -2191,9 +2199,13 @@ this means that there is a possibility that the callback will not be called unde -> Stability: 1.1 - Active Development +> Stability: 1.2 - Release candidate * `ref` {Object | Function} The reference to the resource that was registered previously. diff --git a/lib/internal/process/finalization.js b/lib/internal/process/finalization.js index 5efc2c8d78f9..62b8cb563ba3 100644 --- a/lib/internal/process/finalization.js +++ b/lib/internal/process/finalization.js @@ -7,7 +7,6 @@ const { SafeWeakRef, } = primordials; const { validateObject, kValidateObjectAllowFunction } = require('internal/validators'); -const { emitExperimentalWarning } = require('internal/util'); function createFinalization() { /** @@ -97,7 +96,6 @@ function createFinalization() { * @param {Function} fn */ function register(obj, fn) { - emitExperimentalWarning('process.finalization.register'); validateObject(obj, 'obj', kValidateObjectAllowFunction); _register('exit', obj, fn); @@ -110,7 +108,6 @@ function createFinalization() { * @param {Function} fn */ function registerBeforeExit(obj, fn) { - emitExperimentalWarning('process.finalization.registerBeforeExit'); validateObject(obj, 'obj', kValidateObjectAllowFunction); _register('beforeExit', obj, fn); @@ -121,7 +118,6 @@ function createFinalization() { * @param {object} obj */ function unregister(obj) { - emitExperimentalWarning('process.finalization.unregister'); if (!registry) { return; } From e6662cb904d98d1d716cca9a4c37af35eeda7911 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 01:40:48 +0000 Subject: [PATCH 58/98] stream: fix Utf8Stream flush handling Fix several issues with `Utf8Stream` flushing: * `flush()` now writes buffered data regardless of `minLength` and invokes the callback only after pending writes complete, including when `minLength` is zero and a write is in flight. * Multiple concurrent `flush()` calls are tracked correctly and `end()` waits for pending flushes before closing. * `flushSync()` throws `ERR_INVALID_STATE` if called while an asynchronous write is in progress instead of corrupting output. * Periodic flushes no longer stack up while a flush is pending. * `fsync` is skipped for stdout/stderr file descriptors. * `reopen()`, `end()` and `destroy()` behave correctly when the stream is destroyed while still opening. Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/66473 Reviewed-By: Matteo Collina --- doc/api/fs.md | 11 +- lib/internal/streams/fast-utf8-stream.js | 345 ++++++++++++++---- .../test-fastutf8stream-flush-mocks.js | 53 +++ .../test-fastutf8stream-flush-sync.js | 152 +++++++- test/parallel/test-fastutf8stream-flush.js | 167 +++++++++ .../test-fastutf8stream-periodicflush.js | 52 +++ 6 files changed, 702 insertions(+), 78 deletions(-) diff --git a/doc/api/fs.md b/doc/api/fs.md index 351039a6a00a..b4af15dae3f4 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -8508,13 +8508,20 @@ Close the stream gracefully, flushing the internal buffer before closing. * `callback` {Function} * `err` {Error|null} An error if the flush failed, otherwise `null`. -Writes the current buffer to the file if a write was not in progress. Do -nothing if `minLength` is zero or if it is already writing. +Writes the current buffer to the file. The callback is invoked after pending +writes complete and, unless the `fsync` option is enabled, after the data has +been synchronized with `fs.fsync()`. `fsync` errors indicating that the file +descriptor does not support synchronization (for example, a pipe or a TTY) +are ignored. #### `utf8Stream.flushSync()` Flushes the buffered data synchronously. This is a costly operation. +If an asynchronous write is in flight when `flushSync()` is called, only the +data queued after it is written, and it may reach the file before the data +of the in-flight write. + #### `utf8Stream.fsync` * {boolean} Whether the stream is performing a `fs.fsyncSync()` after every diff --git a/lib/internal/streams/fast-utf8-stream.js b/lib/internal/streams/fast-utf8-stream.js index d97f1cae3257..004ff14781f1 100644 --- a/lib/internal/streams/fast-utf8-stream.js +++ b/lib/internal/streams/fast-utf8-stream.js @@ -6,7 +6,9 @@ const { ArrayPrototypePush, + ArrayPrototypeUnshift, MathMax, + Symbol, SymbolDispose, } = primordials; @@ -28,6 +30,7 @@ const EventEmitter = require('events'); const path = require('path'); const { clearInterval, + clearTimeout, setInterval, setTimeout, } = require('timers'); @@ -58,6 +61,38 @@ const kMaxWrite = 16 * 1024; const kContentModeBuffer = 'buffer'; const kContentModeUtf8 = 'utf8'; const kNullPrototype = { __proto__: null }; +const kFlush = Symbol('kFlush'); + +// States returned by #beginFlushSync(). +// No write is in flight. +const kFlushSyncIdle = 0; +// An EAGAIN/EBUSY retry timer was pending and has been cancelled. Nothing is +// in flight, so the remainder of #writingBuf can be written synchronously. +const kFlushSyncRetry = 1; +// Called from a 'write' listener. The completed write has been released and +// nothing is in flight, so the remainder of #writingBuf can be written +// synchronously. +const kFlushSyncInWriteEvent = 2; +// An asynchronous fs.write() of #writingBuf is in flight. Only the queued +// buffers can be written; ordering relative to the in-flight chunk is not +// guaranteed. +const kFlushSyncInFlight = 3; + +// fsync() error codes meaning the file descriptor cannot be synchronized +// (e.g. it is a pipe, socket, or TTY) or has already been closed. These are +// not treated as flush failures. +function isIgnorableFsyncError(err) { + switch (err?.code) { + case 'EBADF': + case 'EINVAL': + case 'ENOTSUP': + case 'EOPNOTSUPP': + case 'EROFS': + return true; + default: + return false; + } +} // Utf8Stream is a port of the original SonicBoom module // (https://github.com/pinojs/sonic-boom) that provides a fast and efficient @@ -71,7 +106,11 @@ class Utf8Stream extends EventEmitter { #ending = false; #reopening = false; #asyncDrainScheduled = false; - #flushPending = false; + #flushPending = 0; + #flushInProgress = 0; + #emittingFlush = false; + #emittingWrite = false; + #retryTimer = null; #hwm = 16387; // 16 KB #file = null; #destroyed = false; @@ -233,7 +272,11 @@ class Utf8Stream extends EventEmitter { }); if (this.#periodicFlush !== 0) { - this.#periodicFlushTimer = setInterval(() => this.flush(null), this.#periodicFlush); + this.#periodicFlushTimer = setInterval(() => { + if (this.#flushPending === 0 && this.#flushInProgress === 0) { + this.flush(); + } + }, this.#periodicFlush); this.#periodicFlushTimer.unref(); } } @@ -268,7 +311,9 @@ class Utf8Stream extends EventEmitter { } if (this.#opening) { - this.once('ready', () => this.reopen(file)); + this.once('ready', () => { + if (!this.#destroyed) this.reopen(file); + }); return; } @@ -311,7 +356,7 @@ class Utf8Stream extends EventEmitter { if (this.#opening) { this.once('ready', () => { - this.end(); + if (!this.#destroyed) this.end(); }); return; } @@ -329,7 +374,7 @@ class Utf8Stream extends EventEmitter { if (this.#len > 0 && this.#fd >= 0) { this.#actualWrite(); } else { - this.#actualClose(); + this.#finishEnding(); } } @@ -337,7 +382,11 @@ class Utf8Stream extends EventEmitter { if (this.#destroyed) { return; } + const opening = this.#opening; this.#actualClose(); + if (opening && this.#flushPending > 0) { + this.emit('error', new ERR_INVALID_STATE('Utf8Stream is destroyed')); + } } /** @type {number} */ @@ -403,7 +452,10 @@ class Utf8Stream extends EventEmitter { } } else { // Let's give the destination some time to process the chunk. - setTimeout(() => this.#fsWrite(), BUSY_WRITE_TIMEOUT); + this.#retryTimer = setTimeout(() => { + this.#retryTimer = null; + this.#fsWrite(); + }, BUSY_WRITE_TIMEOUT); } } else { this.#writing = false; @@ -420,11 +472,20 @@ class Utf8Stream extends EventEmitter { this.#writeRetries = 0; } - this.emit('write', n); const releasedBufObj = releaseWritingBuf(this.#writingBuf, this.#len, n); this.#len = releasedBufObj.len; this.#writingBuf = releasedBufObj.writingBuf; + // Emit 'write' after the written bytes have been released so that a + // listener calling flushSync() sees a consistent state with no I/O in + // flight. flushSync() may consume the remainder of #writingBuf. + this.#emittingWrite = true; + try { + this.emit('write', n); + } finally { + this.#emittingWrite = false; + } + if (this.#writingBuf.length) { if (!this.#sync) { this.#fsWrite(); @@ -444,6 +505,18 @@ class Utf8Stream extends EventEmitter { } } + if (this.#destroyed) { + this.#writing = false; + if (this.#flushPending > 0) { + if (this.#len === 0) { + this.#scheduleDrain(); + } else { + this.emit('error', new ERR_INVALID_STATE('Utf8Stream is destroyed')); + } + } + return; + } + if (this.#fsync) { this.#fs.fsyncSync(this.#fd); } @@ -453,25 +526,19 @@ class Utf8Stream extends EventEmitter { this.#writing = false; this.#reopening = false; this.reopen(); - } else if (len > this.#minLength) { + } else if (len > 0 && + (len > this.#minLength || this.#flushPending)) { this.#actualWrite(); } else if (this.#ending) { if (len > 0) { this.#actualWrite(); } else { this.#writing = false; - this.#actualClose(); + this.#finishEnding(); } } else { this.#writing = false; - if (this.#sync) { - if (!this.#asyncDrainScheduled) { - this.#asyncDrainScheduled = true; - process.nextTick(() => this.#emitDrain()); - } - } else { - this.emit('drain'); - } + this.#scheduleDrain(); } } @@ -520,12 +587,13 @@ class Utf8Stream extends EventEmitter { } // start - if ((!this.#writing && this.#len > this.#minLength) || this.#flushPending) { + if (!this.#writing && + (this.#len > this.#minLength || this.#flushPending)) { this.#actualWrite(); } else if (reopening && !this.#writing) { // Do not emit 'drain' if a 'ready' listener started a write: // #release() will emit the real 'drain' when that write completes. - process.nextTick(() => this.emit('drain')); + process.nextTick(() => this.#emitDrain()); } }; @@ -551,16 +619,59 @@ class Utf8Stream extends EventEmitter { } } + #scheduleDrain() { + if (this.#sync) { + if (!this.#asyncDrainScheduled) { + this.#asyncDrainScheduled = true; + process.nextTick(() => this.#emitDrain()); + } + } else { + this.#emitDrain(); + } + } + #emitDrain() { + this.#asyncDrainScheduled = false; + if (this.#writing) return; + this.#emitFlush(); + if (this.#writing || this.#destroyed) return; const hasListeners = this.listenerCount('drain') > 0; if (!hasListeners) return; - this.#asyncDrainScheduled = false; this.emit('drain'); } + #finishEnding() { + if (!this.#ending || this.#writing || this.#destroyed) return; + if (this.#len > 0) { + this.#actualWrite(); + return; + } + if (this.#flushPending > 0) this.#emitFlush(); + if (this.#flushPending === 0 && this.#flushInProgress === 0 && + !this.#writing && this.#len === 0) { + this.#actualClose(); + } + } + + #emitFlush() { + if (this.#emittingFlush) return; + this.#emittingFlush = true; + try { + this.emit(kFlush); + } finally { + this.#emittingFlush = false; + } + } + #actualClose() { + if (this.#destroyed) return; + if (this.#fd === -1) { - this.once('ready', () => this.#actualClose()); + this.#destroyed = true; + this.once('ready', () => { + this.#destroyed = false; + this.#actualClose(); + }); return; } @@ -636,7 +747,12 @@ class Utf8Stream extends EventEmitter { } } - #flushBufferSync() { + /** + * Validates that flushSync() can run and determines how it must interact + * with any write already in progress. + * @returns {number} One of the kFlushSync* states. + */ + #beginFlushSync() { if (this.#destroyed) { throw new ERR_INVALID_STATE('Utf8Stream is destroyed'); } @@ -645,51 +761,114 @@ class Utf8Stream extends EventEmitter { throw new ERR_INVALID_STATE('Invalid file descriptor'); } - if (!this.#writing && this.#writingBuf.length > 0) { - this.#bufs.unshift([this.#writingBuf]); - this.#writingBuf = kEmptyBuffer; + // While reopening, #writing is set but no write is in flight and #fd still + // refers to the previous file, which is only closed once the new file is + // ready. Flush the buffered data to it. + if (!this.#writing || this.#opening) { + return kFlushSyncIdle; } - let buf = kEmptyBuffer; - while (this.#bufs.length || buf.length) { - if (buf.length <= 0) { - buf = mergeBuf(this.#bufs[0], this.#lens[0]); + if (this.#retryTimer !== null) { + clearTimeout(this.#retryTimer); + this.#retryTimer = null; + return kFlushSyncRetry; + } + + if (this.#emittingWrite) { + return kFlushSyncInWriteEvent; + } + + return kFlushSyncInFlight; + } + + /** + * Called after flushSync() has taken over a write whose retry timer it + * cancelled. Resumes whatever #release() would have done once that write + * completed. + */ + #endFlushSyncRetry() { + this.#writing = false; + process.nextTick(() => { + if (this.#destroyed || this.#writing) return; + if (this.#reopening) { + this.#reopening = false; + this.reopen(); + } else if (this.#ending) { + this.#finishEnding(); + } else if (this.#len > 0 && + (this.#len > this.#minLength || this.#flushPending)) { + this.#actualWrite(); + } else { + this.#emitDrain(); } - try { - const n = this.#fs.writeSync(this.#fd, buf); - this.#writeRetries = 0; - buf = buf.subarray(n); - this.#len = MathMax(this.#len - n, 0); + }); + } + + #flushBufferSync() { + const state = this.#beginFlushSync(); + + // Unless an fs.write() of #writingBuf is in flight, its unwritten + // remainder must be written first to preserve ordering. + if (state !== kFlushSyncInFlight && this.#writingBuf.length > 0) { + ArrayPrototypeUnshift(this.#bufs, [this.#writingBuf]); + ArrayPrototypeUnshift(this.#lens, this.#writingBuf.length); + this.#writingBuf = kEmptyBuffer; + } + + try { + let buf = kEmptyBuffer; + while (this.#bufs.length || buf.length) { if (buf.length <= 0) { - this.#bufs.shift(); - this.#lens.shift(); - } - } catch (err) { - const shouldRetry = err.code === 'EAGAIN' || err.code === 'EBUSY'; - if (shouldRetry) { - this.#writeRetries++; - } - const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries; - if (!shouldRetry || retriesExhausted || !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) { - throw err; + buf = mergeBuf(this.#bufs[0], this.#lens[0]); } + try { + const n = this.#fs.writeSync(this.#fd, buf); + this.#writeRetries = 0; + buf = buf.subarray(n); + this.#len = MathMax(this.#len - n, 0); + if (buf.length <= 0) { + this.#bufs.shift(); + this.#lens.shift(); + } + } catch (err) { + const shouldRetry = err.code === 'EAGAIN' || err.code === 'EBUSY'; + if (shouldRetry) { + this.#writeRetries++; + } + const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries; + if (!shouldRetry || retriesExhausted || !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) { + throw err; + } - sleep(BUSY_WRITE_TIMEOUT); + sleep(BUSY_WRITE_TIMEOUT); + } } + } finally { + if (state === kFlushSyncRetry) this.#endFlushSyncRetry(); } } #flushSyncUtf8() { - if (this.#destroyed) { - throw new ERR_INVALID_STATE('Utf8Stream is destroyed'); + const state = this.#beginFlushSync(); + + try { + this.#flushSyncUtf8Bufs(state); + } finally { + if (state === kFlushSyncRetry) this.#endFlushSyncRetry(); } - if (this.#fd < 0) { - throw new ERR_INVALID_STATE('Invalid file descriptor'); + try { + this.#fs.fsyncSync(this.#fd); + } catch { + // Skip the error. The fd might not support fsync. } + } - if (!this.#writing && this.#writingBuf.length > 0) { - this.#bufs.unshift(this.#writingBuf); + #flushSyncUtf8Bufs(state) { + // Unless an fs.write() of #writingBuf is in flight, its unwritten + // remainder must be written first to preserve ordering. + if (state !== kFlushSyncInFlight && this.#writingBuf.length > 0) { + ArrayPrototypeUnshift(this.#bufs, this.#writingBuf); this.#writingBuf = ''; } @@ -720,46 +899,62 @@ class Utf8Stream extends EventEmitter { sleep(BUSY_WRITE_TIMEOUT); } } - - try { - this.#fs.fsyncSync(this.#fd); - } catch { - // Skip the error. The fd might not support fsync. - } } #callFlushCallbackOnDrain(cb) { - this.#flushPending = true; + this.#flushPending++; + let waiting = true; + let completed = false; + let flushing = false; + const stopWaiting = () => { + if (!waiting) return false; + waiting = false; + this.off(kFlush, onDrain); + this.off('error', onError); + return true; + }; + const complete = (err, finish = true) => { + if (completed) return; + completed = true; + if (flushing) this.#flushInProgress--; + try { + cb(err); + } finally { + if (finish) this.#finishEnding(); + } + }; const onDrain = () => { + if (!stopWaiting()) return; + this.#flushPending--; + this.#flushInProgress++; + flushing = true; // Only if _fsync is false to avoid double fsync if (!this.#fsync && !this.#destroyed) { try { this.#fs.fsync(this.#fd, (err) => { - this.#flushPending = false; - // If the fd is closed, we ignore the error. - if (err?.code === 'EBADF') { - cb(); + // Ignore errors meaning the fd is closed or cannot be synced + // (e.g. stdout/stderr attached to a pipe or TTY). A regular + // file, including a redirected stdout/stderr, is still synced. + if (isIgnorableFsyncError(err)) { + complete(); return; } - cb(err); + complete(err); }); } catch (err) { - this.#flushPending = false; - cb(err); + complete(err); } } else { - this.#flushPending = false; - cb(); + complete(); } - this.off('error', onError); }; const onError = (err) => { - this.#flushPending = false; - cb(err); - this.off('drain', onDrain); + if (!stopWaiting()) return; + this.#flushPending--; + complete(err, false); }; - this.once('drain', onDrain); + this.once(kFlush, onDrain); this.once('error', onError); } @@ -776,7 +971,7 @@ class Utf8Stream extends EventEmitter { throw error; } - if (this.#minLength <= 0) { + if (this.#minLength <= 0 && !this.#writing && this.#len === 0) { cb?.(); return; } @@ -810,7 +1005,7 @@ class Utf8Stream extends EventEmitter { throw error; } - if (this.#minLength <= 0) { + if (this.#minLength <= 0 && !this.#writing && this.#len === 0) { cb?.(); return; } diff --git a/test/parallel/test-fastutf8stream-flush-mocks.js b/test/parallel/test-fastutf8stream-flush-mocks.js index 5d8e19fb1f92..3c6ad039cf4e 100644 --- a/test/parallel/test-fastutf8stream-flush-mocks.js +++ b/test/parallel/test-fastutf8stream-flush-mocks.js @@ -3,9 +3,13 @@ const common = require('../common'); const tmpdir = require('../common/tmpdir'); const assert = require('node:assert'); +const { spawnSync } = require('node:child_process'); const { + closeSync, openSync, + fsync, fsyncSync, + readFileSync, writeSync, write, } = require('node:fs'); @@ -13,6 +17,23 @@ const { join } = require('node:path'); const { Utf8Stream } = require('node:fs'); const { isMainThread } = require('node:worker_threads'); +if (process.argv[2] === 'child-stdout') { + // stdout is redirected to a regular file by the parent. + const stream = new Utf8Stream({ + fd: 1, + minLength: 4096, + fs: { + fsync: common.mustCall((fd, cb) => { + assert.strictEqual(fd, 1); + fsync(fd, cb); + }), + }, + }); + stream.write('hello world\n'); + stream.flush(common.mustSucceed()); + return; +} + tmpdir.refresh(); if (isMainThread) { process.umask(0o000); @@ -27,6 +48,38 @@ function getTempFile() { runTests(false); runTests(true); +// Errors from fsync meaning the fd cannot be synchronized (e.g. a pipe or +// TTY) or is already closed do not fail the flush. +for (const code of ['EBADF', 'EINVAL', 'ENOTSUP', 'EOPNOTSUPP', 'EROFS']) { + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const fsOverride = { + fsync: common.mustCall((fd, cb) => { + const err = new Error(code); + err.code = code; + process.nextTick(cb, err); + }, 2), + }; + const stream = new Utf8Stream({ fd, minLength: 4096, fs: fsOverride }); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write('hello world\n')); + stream.flush(common.mustSucceed(() => stream.end())); + })); +} + +// stdout redirected to a regular file is still fsynced by flush(). +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const child = spawnSync(process.execPath, [__filename, 'child-stdout'], { + stdio: ['ignore', fd, 'pipe'], + }); + closeSync(fd); + assert.strictEqual(child.status, 0, child.stderr.toString()); + assert.strictEqual(readFileSync(dest, 'utf8'), 'hello world\n'); +} + function runTests(sync) { { diff --git a/test/parallel/test-fastutf8stream-flush-sync.js b/test/parallel/test-fastutf8stream-flush-sync.js index a48dac8e08e9..6c7d728fec59 100644 --- a/test/parallel/test-fastutf8stream-flush-sync.js +++ b/test/parallel/test-fastutf8stream-flush-sync.js @@ -67,7 +67,7 @@ function runTests(sync) { const stream = new Utf8Stream({ fd, sync: false, - minLength: 0, + minLength: 1000, fs: fsOverride, }); @@ -85,6 +85,156 @@ function runTests(sync) { })); } +function toData(contentMode, text) { + return contentMode === 'buffer' ? Buffer.from(text) : text; +} + +// flushSync() while an asynchronous write is in flight writes the queued +// data without throwing, and the in-flight write still completes. +for (const contentMode of ['utf8', 'buffer']) { + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + let completeWrite; + const fsOverride = { + write: common.mustCall((fd, buf, ...args) => { + const cb = args.pop(); + completeWrite = () => cb(null, writeSync(fd, buf)); + }), + }; + const stream = new Utf8Stream({ + contentMode, + fd, + minLength: 0, + sync: false, + fs: fsOverride, + }); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write(toData(contentMode, 'in flight\n'))); + assert.strictEqual(stream.writing, true); + assert.ok(stream.write(toData(contentMode, 'queued\n'))); + stream.flushSync(); + assert.strictEqual(readFileSync(dest, 'utf8'), 'queued\n'); + + stream.flush(common.mustSucceed(() => { + stream.on('finish', common.mustCall(() => { + assert.strictEqual(readFileSync(dest, 'utf8'), 'queued\nin flight\n'); + })); + stream.end(); + })); + completeWrite(); + })); +} + +// flushSync() while waiting to retry an EAGAIN write cancels the retry and +// writes everything synchronously, in order. +for (const contentMode of ['utf8', 'buffer']) { + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const fsOverride = { + write: common.mustCallAtLeast((fd, buf, ...args) => { + const cb = args.pop(); + const err = new Error('EAGAIN'); + err.code = 'EAGAIN'; + process.nextTick(cb, err); + }), + }; + const stream = new Utf8Stream({ + contentMode, + fd, + minLength: 0, + sync: false, + fs: fsOverride, + }); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write(toData(contentMode, 'hello\n'))); + // Wait for the EAGAIN to schedule the retry. + setImmediate(common.mustCall(() => { + assert.strictEqual(stream.writing, true); + assert.ok(stream.write(toData(contentMode, 'world\n'))); + // A pending flush() completes once flushSync() takes over the write. + stream.flush(common.mustSucceed(() => { + stream.on('finish', common.mustCall(() => { + assert.strictEqual(readFileSync(dest, 'utf8'), 'hello\nworld\n'); + })); + stream.end(); + })); + stream.flushSync(); + assert.strictEqual(stream.writing, false); + assert.strictEqual(readFileSync(dest, 'utf8'), 'hello\nworld\n'); + })); + })); +} + +// flushSync() from a 'write' listener writes the unwritten remainder of a +// partial write before newly queued data. +for (const sync of [true, false]) { + for (const contentMode of ['utf8', 'buffer']) { + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + let partial = true; + const partialWrite = (fd, buf) => { + if (partial) { + partial = false; + return writeSync(fd, Buffer.from(buf).subarray(0, 5)); + } + return writeSync(fd, buf); + }; + const fsOverride = { + writeSync: (fd, buf, ...args) => partialWrite(fd, buf), + write: (fd, buf, ...args) => { + const cb = args.pop(); + process.nextTick(cb, null, partialWrite(fd, buf)); + }, + }; + const stream = new Utf8Stream({ + contentMode, + fd, + minLength: 0, + sync, + fs: fsOverride, + }); + + stream.once('write', common.mustCall((n) => { + assert.strictEqual(n, 5); + assert.strictEqual(stream.writing, true); + assert.ok(stream.write(toData(contentMode, 'next\n'))); + stream.flushSync(); + assert.strictEqual(readFileSync(dest, 'utf8'), 'hello world\nnext\n'); + })); + + stream.on('ready', common.mustCall(() => { + assert.ok(stream.write(toData(contentMode, 'hello world\n'))); + stream.on('finish', common.mustCall(() => { + assert.strictEqual(readFileSync(dest, 'utf8'), 'hello world\nnext\n'); + })); + stream.end(); + })); + } +} + +// flushSync() while reopening writes the buffered data to the previous file, +// which stays open until the new file is ready. +{ + const dest = getTempFile(); + const stream = new Utf8Stream({ dest, minLength: 4096, sync: false }); + + stream.once('ready', common.mustCall(() => { + assert.ok(stream.write('before reopen\n')); + stream.reopen(); + assert.strictEqual(stream.writing, true); + stream.flushSync(); + assert.strictEqual(readFileSync(dest, 'utf8'), 'before reopen\n'); + stream.once('ready', common.mustCall(() => { + stream.on('finish', common.mustCall(() => { + assert.strictEqual(readFileSync(dest, 'utf8'), 'before reopen\n'); + })); + stream.end(); + })); + })); +} + { const dest = getTempFile(); const fd = openSync(dest, 'w'); diff --git a/test/parallel/test-fastutf8stream-flush.js b/test/parallel/test-fastutf8stream-flush.js index 60238ed76908..14d77166c4c8 100644 --- a/test/parallel/test-fastutf8stream-flush.js +++ b/test/parallel/test-fastutf8stream-flush.js @@ -53,6 +53,173 @@ runTests(true); stream.on('ready', common.mustCall()); } +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const stream = new Utf8Stream({ fd, minLength: 10, sync: false }); + let flushed = false; + + assert.ok(stream.write('asynchronous flush\n')); + assert.ok(stream.write('queued\n')); + assert.strictEqual(stream.writing, true); + stream.flush(common.mustSucceed(() => { + flushed = true; + assert.strictEqual(stream.writing, false); + readFile(dest, 'utf8', common.mustSucceed((data) => { + assert.strictEqual(data, 'asynchronous flush\nqueued\n'); + stream.end(); + })); + })); + assert.strictEqual(flushed, false); +} + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const stream = new Utf8Stream({ fd, minLength: 0, sync: false }); + + assert.ok(stream.write('flush before end\n')); + stream.flush(common.mustSucceed()); + stream.on('finish', common.mustCall(() => { + readFile(dest, 'utf8', common.mustSucceed((data) => { + assert.strictEqual(data, 'flush before end\n'); + })); + })); + stream.end(); +} + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const writeError = new Error('write failed'); + const stream = new Utf8Stream({ + fd, + minLength: 0, + sync: false, + fs: { + write: common.mustCall((_fd, _data, _encoding, callback) => { + process.nextTick(callback, writeError); + }), + }, + }); + + stream.on('error', common.mustCall((error) => { + assert.strictEqual(error, writeError); + })); + assert.ok(stream.write('failed write before end\n')); + stream.flush(common.mustCall((error) => { + assert.strictEqual(error, writeError); + })); + stream.end(); +} + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + let fsyncCalls = 0; + const stream = new Utf8Stream({ + fd, + minLength: 10, + sync: false, + fs: { + fsync: common.mustCall((_fd, callback) => { + fsyncCalls++; + if (fsyncCalls === 1) { + assert.ok(stream.write('late\n')); + } + process.nextTick(callback); + }, 2), + }, + }); + + assert.ok(stream.write('initial write\n')); + stream.flush(common.mustSucceed()); + stream.on('finish', common.mustCall(() => { + readFile(dest, 'utf8', common.mustSucceed((data) => { + assert.strictEqual(data, 'initial write\nlate\n'); + })); + })); + stream.end(); +} + +{ + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const flushError = new Error('flush failed'); + let fsyncCalls = 0; + const stream = new Utf8Stream({ + fd, + minLength: 0, + sync: false, + fs: { + fsync: common.mustCall((_fd, callback) => { + fsyncCalls++; + process.nextTick(callback, fsyncCalls === 1 ? flushError : null); + }, 2), + }, + }); + + assert.ok(stream.write('failed flush before end\n')); + stream.flush(common.mustCall((error) => { + assert.strictEqual(error, flushError); + })); + stream.on('close', common.mustCall()); + stream.end(); +} + +{ + const dest = getTempFile(); + const stream = new Utf8Stream({ + dest, + fs: { + open(...args) { + setImmediate(() => open(...args)); + }, + }, + }); + + stream.flush(common.mustSucceed()); + stream.on('close', common.mustCall()); + stream.end(); +} + +{ + const dest = getTempFile(); + const stream = new Utf8Stream({ + dest, + fs: { + open(...args) { + setImmediate(() => open(...args)); + }, + }, + }); + + stream.flush(common.mustCall((error) => { + assert.strictEqual(error?.code, 'ERR_INVALID_STATE'); + })); + stream.on('close', common.mustCall()); + stream.destroy(); + stream.flush(common.mustCall((error) => { + assert.strictEqual(error?.code, 'ERR_INVALID_STATE'); + })); +} + +{ + const dest = getTempFile(); + const stream = new Utf8Stream({ + dest, + fs: { + open(...args) { + setImmediate(() => open(...args)); + }, + }, + }); + + stream.on('close', common.mustCall()); + stream.end(); + stream.destroy(); +} + function runTests(sync) { { const dest = getTempFile(); diff --git a/test/parallel/test-fastutf8stream-periodicflush.js b/test/parallel/test-fastutf8stream-periodicflush.js index f7029a213502..4889fa8b19eb 100644 --- a/test/parallel/test-fastutf8stream-periodicflush.js +++ b/test/parallel/test-fastutf8stream-periodicflush.js @@ -75,4 +75,56 @@ function runTests(sync) { stream.destroy(); } + + { + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + const stream = new Utf8Stream({ + fd, + sync, + minLength: 5000, + periodicFlush: common.platformTimeout(10), + }); + const timeout = setTimeout( + common.mustNotCall('periodic flush did not complete'), + common.platformTimeout(1000), + ); + + stream.once('drain', common.mustCall(() => { + clearTimeout(timeout); + stream.destroy(); + readFile(dest, 'utf8', common.mustSucceed((data) => { + assert.strictEqual(data, 'periodic flush\n'); + })); + })); + assert.ok(stream.write('periodic flush\n')); + } + + if (!sync) { + const dest = getTempFile(); + const fd = openSync(dest, 'w'); + let fsyncCalls = 0; + const stream = new Utf8Stream({ + fd, + minLength: 5000, + periodicFlush: common.platformTimeout(10), + fs: { + fsync: common.mustCall((_fd, callback) => { + fsyncCalls++; + if (fsyncCalls === 1) { + setTimeout(common.mustCall(() => { + assert.strictEqual(fsyncCalls, 1); + stream.destroy(); + callback(); + }), common.platformTimeout(50)); + } else { + process.nextTick(callback); + } + }, 2), + }, + }); + + stream.on('close', common.mustCall()); + assert.ok(stream.write('slow fsync\n')); + } } From 2f498f08fed06a7ada43a0f1697dcb4360a218f3 Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Sat, 10 Oct 2026 09:27:17 -0300 Subject: [PATCH 59/98] benchmark: add util.parseArgs benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66599 Reviewed-By: Ulises Gascón Reviewed-By: Vinícius Lourenço Claro Cardoso Reviewed-By: Luigi Pinca --- benchmark/util/parse-args.js | 67 ++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 benchmark/util/parse-args.js diff --git a/benchmark/util/parse-args.js b/benchmark/util/parse-args.js new file mode 100644 index 000000000000..ac3737281920 --- /dev/null +++ b/benchmark/util/parse-args.js @@ -0,0 +1,67 @@ +'use strict'; + +const util = require('node:util'); +const assert = require('node:assert'); +const common = require('../common'); + +const inputs = { + 'short-flags': { + args: ['-a', '-b', '-c'], + options: { + alpha: { type: 'boolean', short: 'a' }, + beta: { type: 'boolean', short: 'b' }, + gamma: { type: 'boolean', short: 'c' }, + }, + }, + 'long-flags': { + args: ['--foo', '--bar', '--baz'], + options: { + foo: { type: 'boolean' }, + bar: { type: 'boolean' }, + baz: { type: 'boolean' }, + }, + }, + 'values': { + args: ['--foo=1', '--bar', '2', '-b', '3'], + options: { + foo: { type: 'string' }, + bar: { type: 'string' }, + baz: { type: 'string', short: 'b' }, + }, + }, + 'non-strict': { + args: ['--foo', '--unknown=value', '-x', 'file.js', 'other.js'], + options: { + foo: { type: 'boolean' }, + }, + strict: false, + }, + 'tokens': { + args: ['-ab', '--foo=1', '--bar', '2', 'file.js'], + options: { + alpha: { type: 'boolean', short: 'a' }, + beta: { type: 'boolean', short: 'b' }, + foo: { type: 'string' }, + bar: { type: 'string' }, + }, + allowPositionals: true, + tokens: true, + }, +}; + +const bench = common.createBenchmark(main, { + n: [1e5], + type: Object.keys(inputs), +}); + +function main({ n, type }) { + const config = inputs[type]; + let noDead; + + bench.start(); + for (let i = 0; i < n; i++) { + noDead = util.parseArgs(config); + } + bench.end(n); + assert(noDead !== undefined); +} From 115731dda70d04bab0b0f144332274e954d60b73 Mon Sep 17 00:00:00 2001 From: Roberto Bianchi Date: Sat, 10 Oct 2026 15:02:45 +0200 Subject: [PATCH 60/98] http2: speed up response getHeaders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copy validated header names directly into the shallow result instead of using ObjectAssign on the internal null-prototype map. Signed-off-by: Roberto Bianchi PR-URL: https://github.com/nodejs/node/pull/66592 Reviewed-By: Tim Perry Reviewed-By: Matteo Collina Reviewed-By: Gürgün Dayıoğlu --- benchmark/http2/get-headers.js | 46 +++++++++++ lib/internal/http2/compat.js | 8 +- .../parallel/test-http2-compat-get-headers.js | 80 +++++++++++++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 benchmark/http2/get-headers.js create mode 100644 test/parallel/test-http2-compat-get-headers.js diff --git a/benchmark/http2/get-headers.js b/benchmark/http2/get-headers.js new file mode 100644 index 000000000000..a0732d92d7c9 --- /dev/null +++ b/benchmark/http2/get-headers.js @@ -0,0 +1,46 @@ +'use strict'; + +const common = require('../common.js'); +const assert = require('node:assert'); +const http2 = require('node:http2'); + +const bench = common.createBenchmark(main, { + n: [1e6], + headers: [0, 1, 4, 16, 64], +}); + +function main({ n, headers }) { + const server = http2.createServer(); + server.on('request', (request, response) => { + const expected = { __proto__: null }; + for (let index = 0; index < headers; index++) { + const name = `x-header-${index}`; + response.setHeader(name, 'value'); + expected[name] = 'value'; + } + + for (let index = 0; index < 1e4; index++) { + response.getHeaders(); + } + + let result; + bench.start(); + for (let index = 0; index < n; index++) { + result = response.getHeaders(); + } + bench.end(n); + assert.deepStrictEqual(result, expected); + response.end(); + }); + + server.listen(0, '127.0.0.1', () => { + const client = http2.connect(`http://127.0.0.1:${server.address().port}`); + const request = client.request(); + request.resume(); + request.on('end', () => { + client.close(); + server.close(); + }); + request.end(); + }); +} diff --git a/lib/internal/http2/compat.js b/lib/internal/http2/compat.js index 1df6a6820af9..280314982890 100644 --- a/lib/internal/http2/compat.js +++ b/lib/internal/http2/compat.js @@ -605,7 +605,13 @@ class Http2ServerResponse extends Stream { getHeaders() { const headers = { __proto__: null }; - return ObjectAssign(headers, this[kHeaders]); + const source = this[kHeaders]; + const keys = ObjectKeys(source); + for (let index = 0; index < keys.length; index++) { + const key = keys[index]; + headers[key] = source[key]; + } + return headers; } hasHeader(name) { diff --git a/test/parallel/test-http2-compat-get-headers.js b/test/parallel/test-http2-compat-get-headers.js new file mode 100644 index 000000000000..f98ceca1db14 --- /dev/null +++ b/test/parallel/test-http2-compat-get-headers.js @@ -0,0 +1,80 @@ +'use strict'; + +const common = require('../common'); +if (!common.hasCrypto) + common.skip('missing crypto'); + +const assert = require('node:assert'); +const http2 = require('node:http2'); + +const server = http2.createServer(common.mustCall((request, response) => { + const empty = response.getHeaders(); + assert.strictEqual(Object.getPrototypeOf(empty), null); + assert.deepStrictEqual(Object.keys(empty), []); + assert.notStrictEqual(response.getHeaders(), empty); + + const cookies = ['first=value', 'second=value']; + response.setHeader('X-First', 'first'); + response.setHeader('10', 'ten'); + response.setHeader('2', 'two'); + response.setHeader('__proto__', 'proto'); + response.setHeader('Constructor', 'constructor'); + response.setHeader('Set-Cookie', cookies); + + const expected = { + '__proto__': null, + '2': 'two', + '10': 'ten', + 'x-first': 'first', + ['__proto__']: 'proto', + 'constructor': 'constructor', + 'set-cookie': cookies, + }; + const headers = response.getHeaders(); + assert.deepStrictEqual(headers, expected); + assert.deepStrictEqual(Object.keys(headers), Object.keys(expected)); + assert.strictEqual(headers['set-cookie'], cookies); + assert.notStrictEqual(response.getHeaders(), headers); + + headers['x-first'] = 'changed'; + headers['x-new'] = 'new'; + delete headers['2']; + assert.deepStrictEqual(response.getHeaders(), expected); + + headers['set-cookie'].push('third=value'); + assert.strictEqual(response.getHeader('set-cookie'), cookies); + assert.strictEqual(response.getHeader('set-cookie').length, 3); + + response.removeHeader('X-First'); + response.setHeader('X-First', 'replacement'); + delete expected['x-first']; + expected['x-first'] = 'replacement'; + assert.deepStrictEqual(response.getHeaders(), expected); + assert.deepStrictEqual(Object.keys(response.getHeaders()), Object.keys(expected)); + + for (const name of response.getHeaderNames()) { + response.removeHeader(name); + } + assert.deepStrictEqual(response.getHeaders(), empty); + assert.notStrictEqual(response.getHeaders(), empty); + + response.setHeader('X-Sent', 'value'); + response.flushHeaders(); + assert.deepStrictEqual(response.getHeaders(), { + '__proto__': null, + 'x-sent': 'value', + ':status': 200, + }); + response.end(); +})); + +server.listen(0, common.mustCall(() => { + const client = http2.connect(`http://localhost:${server.address().port}`); + const request = client.request(); + request.resume(); + request.on('end', common.mustCall(() => { + client.close(); + server.close(); + })); + request.end(); +})); From adef72223a442818019fb18d989bb3a9cd3b9f57 Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Sat, 10 Oct 2026 11:22:00 -0300 Subject: [PATCH 61/98] benchmark: fix buffer option in buffer read/write benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main() read `buf` instead of `buffer`, so every run measured Buffer.allocUnsafeSlow() even when labeled buffer=fast. Also benchmark the slow variant and trim the byteLength matrix. Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66604 Reviewed-By: Ulises Gascón Reviewed-By: Luigi Pinca Reviewed-By: Minwoo Jung --- benchmark/buffers/buffer-read-with-byteLength.js | 8 ++++---- benchmark/buffers/buffer-read.js | 6 +++--- benchmark/buffers/buffer-write.js | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmark/buffers/buffer-read-with-byteLength.js b/benchmark/buffers/buffer-read-with-byteLength.js index 4cbe12ef4fe3..1f6cfdf1490f 100644 --- a/benchmark/buffers/buffer-read-with-byteLength.js +++ b/benchmark/buffers/buffer-read-with-byteLength.js @@ -10,14 +10,14 @@ const types = [ ]; const bench = common.createBenchmark(main, { - buffer: ['fast'], + buffer: ['fast', 'slow'], type: types, n: [1e6], - byteLength: [1, 2, 3, 4, 5, 6], + byteLength: [1, 2, 4, 6], }); -function main({ n, buf, type, byteLength }) { - const buff = buf === 'fast' ? +function main({ n, buffer, type, byteLength }) { + const buff = buffer === 'fast' ? Buffer.alloc(8) : Buffer.allocUnsafeSlow(8); const fn = `read${type}`; diff --git a/benchmark/buffers/buffer-read.js b/benchmark/buffers/buffer-read.js index 2543db7f5d6d..19570a6b9241 100644 --- a/benchmark/buffers/buffer-read.js +++ b/benchmark/buffers/buffer-read.js @@ -20,13 +20,13 @@ const types = [ ]; const bench = common.createBenchmark(main, { - buffer: ['fast'], + buffer: ['fast', 'slow'], type: types, n: [1e6], }); -function main({ n, buf, type }) { - const buff = buf === 'fast' ? +function main({ n, buffer, type }) { + const buff = buffer === 'fast' ? Buffer.alloc(8) : Buffer.allocUnsafeSlow(8); const fn = `read${type}`; diff --git a/benchmark/buffers/buffer-write.js b/benchmark/buffers/buffer-write.js index 25a3a04a130a..66ef97fbae09 100644 --- a/benchmark/buffers/buffer-write.js +++ b/benchmark/buffers/buffer-write.js @@ -27,7 +27,7 @@ const types = [ ]; const bench = common.createBenchmark(main, { - buffer: ['fast'], + buffer: ['fast', 'slow'], type: types, n: [1e6], }); @@ -70,8 +70,8 @@ const byteLength = { writeIntBE: 6, }; -function main({ n, buf, type }) { - const buff = buf === 'fast' ? +function main({ n, buffer, type }) { + const buff = buffer === 'fast' ? Buffer.alloc(8) : Buffer.allocUnsafeSlow(8); const fn = `write${type}`; From 7309ddf204f13cff581cdc39e3748206e5d982a9 Mon Sep 17 00:00:00 2001 From: Barak <26878518+barakor@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:36:38 +0300 Subject: [PATCH 62/98] test_runner: rerun failing todo tests in suites A failing todo subtest does not fail its parent, so the parent suite was recorded as passed in the rerun state file along with its children. On the next run the suite was replayed synthetically and the failing todo subtest was marked as passed instead of being rerun. Do not record a passed test whose descendant failed, so the suite body runs again while passing siblings are still replayed from their own entries. Document that a failing todo test counts as not yet passed. Signed-off-by: Barak <26878518+barakor@users.noreply.github.com> PR-URL: https://github.com/nodejs/node/pull/66605 Reviewed-By: Benjamin Gruenbaum Reviewed-By: Moshe Atlow Reviewed-By: Chemi Atlow --- doc/api/test.md | 2 + lib/internal/test_runner/reporter/rerun.js | 14 +++- test/fixtures/test-runner/rerun-todo.js | 21 ++++++ .../test-runner-test-rerun-failures.js | 70 +++++++++++++++++++ 4 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/test-runner/rerun-todo.js diff --git a/doc/api/test.md b/doc/api/test.md index 158e8e961d5d..d5b42ab8dc0a 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -160,6 +160,8 @@ in this example, there are two run attempts, with two tests defined in `test.js` the first test succeeded on the first attempt, and the second test succeeded on the second attempt. When the `--test-rerun-failures` option is used, the test runner will only run tests that have not yet passed. +A `todo` test that fails counts as not yet passed and is rerun, even though its failure does not +cause the test run to fail. ```bash node --test-rerun-failures /path/to/state/file diff --git a/lib/internal/test_runner/reporter/rerun.js b/lib/internal/test_runner/reporter/rerun.js index 3f9ae102fea3..ba2a1fe39141 100644 --- a/lib/internal/test_runner/reporter/rerun.js +++ b/lib/internal/test_runner/reporter/rerun.js @@ -21,7 +21,13 @@ function reportReruns(previousRuns, globalOptions) { function startTest(data) { const originalSuite = currentSuite; - currentSuite = { __proto__: null, data, parent: currentSuite, children: [] }; + currentSuite = { + __proto__: null, + data, + parent: currentSuite, + children: [], + hasFailedDescendant: false, + }; if (originalSuite?.children) { ArrayPrototypePush(originalSuite.children, currentSuite); } @@ -57,7 +63,11 @@ function reportReruns(previousRuns, globalOptions) { } else { disambiguator[baseIdentifier] = 1; } - if (type === 'test:pass') { + if (type === 'test:fail') { + for (let t = currentTest.parent; t !== null; t = t.parent) { + t.hasFailedDescendant = true; + } + } else if (!currentTest.hasFailedDescendant) { const children = ArrayPrototypeMap(currentTest.children, (child) => child.data); obj[identifier] = { __proto__: null, diff --git a/test/fixtures/test-runner/rerun-todo.js b/test/fixtures/test-runner/rerun-todo.js new file mode 100644 index 000000000000..c8b9fbb99a93 --- /dev/null +++ b/test/fixtures/test-runner/rerun-todo.js @@ -0,0 +1,21 @@ +const { test, describe } = require('node:test'); + +test('top-level todo fails on first attempt', { todo: true }, ({ attempt }) => { + if (attempt < 1) { + throw new Error('This todo test is expected to fail on the first attempt'); + } +}); + +describe('suite with failing todo', () => { + test('nested todo fails on first attempt', { todo: true }, ({ attempt }) => { + if (attempt < 1) { + throw new Error('This todo test is expected to fail on the first attempt'); + } + }); + + test('ok', ({ attempt }) => { + if (attempt > 0) { + throw new Error('Test should not rerun once it has passed'); + } + }); +}); diff --git a/test/parallel/test-runner-test-rerun-failures.js b/test/parallel/test-runner-test-rerun-failures.js index 0af3722fad70..585406ec919c 100644 --- a/test/parallel/test-runner-test-rerun-failures.js +++ b/test/parallel/test-runner-test-rerun-failures.js @@ -283,3 +283,73 @@ test('using `run` api', async () => { await setTimeout(common.platformTimeout(10)); // Wait for the stream to finish processing assert.deepStrictEqual(await getStateFile(), expectedStateFile); }); + +test('failing todo test is rerun even when its parent suite passed', async () => { + const fixturePath = fixtures.path('test-runner', 'rerun-todo.js'); + const fixtureKey = relative(process.cwd(), fixturePath).replaceAll('\\', '/'); + const topLevelTodo = `${fixtureKey}:3:1`; + const suite = `${fixtureKey}:9:1`; + const nestedTodo = `${fixtureKey}:10:3`; + const ok = `${fixtureKey}:16:3`; + + async function runAttempt() { + const results = { __proto__: null, passed: [], failed: [], summaries: [] }; + const stream = run({ files: [fixturePath], rerunFailuresFilePath: stateFile }); + for await (const { type, data } of stream) { + const { name, todo, details } = data; + if (type === 'test:pass') { + results.passed.push({ name, todo, attempt: details.attempt, passed_on_attempt: details.passed_on_attempt }); + } else if (type === 'test:fail') { + results.failed.push({ name, todo, attempt: details.attempt }); + } else if (type === 'test:summary') { + results.summaries.push(data); + } + } + await setTimeout(common.platformTimeout(10)); // Wait for the stream to finish processing + return results; + } + + let results = await runAttempt(); + assert.deepStrictEqual(results.failed, [ + { name: 'top-level todo fails on first attempt', todo: true, attempt: 0 }, + { name: 'nested todo fails on first attempt', todo: true, attempt: 0 }, + ]); + assert.deepStrictEqual(results.passed, [ + { name: 'ok', todo: undefined, attempt: 0, passed_on_attempt: undefined }, + { name: 'suite with failing todo', todo: undefined, attempt: 0, passed_on_attempt: undefined }, + ]); + for (const { success, counts } of results.summaries) { + assert.strictEqual(success, true); + assert.strictEqual(counts.failed, 0); + assert.strictEqual(counts.passed, 1); + assert.strictEqual(counts.todo, 2); + } + let state = await getStateFile(); + assert.strictEqual(state.length, 1); + assert.deepStrictEqual(state[0], { + [ok]: { passed_on_attempt: 0, name: 'ok' }, + }); + + results = await runAttempt(); + assert.deepStrictEqual(results.failed, []); + assert.deepStrictEqual(results.passed, [ + { name: 'top-level todo fails on first attempt', todo: true, attempt: 1, passed_on_attempt: undefined }, + { name: 'nested todo fails on first attempt', todo: true, attempt: 1, passed_on_attempt: undefined }, + { name: 'ok', todo: undefined, attempt: 1, passed_on_attempt: 0 }, + { name: 'suite with failing todo', todo: undefined, attempt: 1, passed_on_attempt: undefined }, + ]); + for (const { success, counts } of results.summaries) { + assert.strictEqual(success, true); + assert.strictEqual(counts.failed, 0); + assert.strictEqual(counts.passed, 1); + assert.strictEqual(counts.todo, 2); + } + state = await getStateFile(); + assert.strictEqual(state.length, 2); + assert.deepStrictEqual(state[1], { + [topLevelTodo]: { passed_on_attempt: 1, name: 'top-level todo fails on first attempt' }, + [nestedTodo]: { passed_on_attempt: 1, name: 'nested todo fails on first attempt' }, + [ok]: { passed_on_attempt: 0, name: 'ok' }, + [suite]: { passed_on_attempt: 1, name: 'suite with failing todo' }, + }); +}); From 30e76bb16a9c8eb00fd2b5a9a329b83d804fe1ee Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Sat, 10 Oct 2026 13:22:11 -0300 Subject: [PATCH 63/98] benchmark: fix input passed to subtle.digest in webcrypto-digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data is already a random Buffer, so passing it through TextEncoder decoded it as UTF-8 and re-encoded it, making subtle.digest hash a different and larger input than createHash. Signed-off-by: RafaelGSS PR-URL: https://github.com/nodejs/node/pull/66608 Reviewed-By: Ulises Gascón Reviewed-By: Luigi Pinca --- benchmark/crypto/webcrypto-digest.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmark/crypto/webcrypto-digest.js b/benchmark/crypto/webcrypto-digest.js index f5ef8e6bc493..e25181a3c3d6 100644 --- a/benchmark/crypto/webcrypto-digest.js +++ b/benchmark/crypto/webcrypto-digest.js @@ -33,8 +33,6 @@ function measureLegacy(n, data, method) { } function measureSubtle(n, data, method) { - const ec = new TextEncoder(); - data = ec.encode(data); const jobs = new Array(n); bench.start(); for (let i = 0; i < n; i++) From 700c60a1cd3a865bf2ca7cb5e5aee49f716fef1c Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:39:27 +0000 Subject: [PATCH 64/98] fs: serialize FileHandle writer() async writes When writer() was created without a `start` offset, every write() and writev() call targeted the file descriptor's current position, and nothing prevented several of them from being in flight at once. Overlapping un-awaited writes then raced on the shared file offset (and partial writes were completed in follow-up syscalls), silently writing data at the wrong offsets while the reported byte count and the final file size still looked correct. Issue async writes one at a time, in call order. A write that is still queued when the writer fails, or whose signal aborts while it is queued, is no longer started. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/fs/promises.js | 29 ++++++++- .../test-fs-promises-file-handle-writer.js | 65 +++++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 7fd37c3c97de..9bd430df795e 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -751,6 +751,11 @@ if (getOptionValue('--experimental-stream-iter')) { let asyncPending = 0; let released = false; const pendingWrites = new SafeSet(); + // Async writes are issued one at a time, in call order. Without this, + // overlapping write() calls would race on the file position: when no + // `start` is given every write targets the fd's current (shared) + // position, and partial writes are completed in follow-up syscalls. + let writeQueue = PromiseResolve(); validateBoolean(autoClose, 'options.autoClose'); @@ -866,6 +871,22 @@ if (getOptionValue('--experimental-stream-iter')) { }); } + function ignoreWriteQueueResult() {} + + // Run `start` once every previously queued write has settled. + function enqueueWrite(start, signal) { + const operation = PromisePrototypeThen(writeQueue, () => { + // A failed writer must not touch the file again, and a write whose + // signal aborted while it was queued must not be started. + if (errored) throw error; + signal?.throwIfAborted(); + return start(); + }); + writeQueue = PromisePrototypeThen(operation, ignoreWriteQueueResult, + ignoreWriteQueueResult); + return operation; + } + function trackOperation(operation) { const { promise, resolve, reject } = PromiseWithResolvers(); const pending = { __proto__: null, reject }; @@ -950,8 +971,9 @@ if (getOptionValue('--experimental-stream-iter')) { if (bytesRemaining > 0) bytesRemaining -= chunk.byteLength; const position = pos; if (pos >= 0) pos += chunk.byteLength; - return trackOperation( - writeAll(chunk, 0, chunk.byteLength, position, signal)); + return trackOperation(enqueueWrite( + () => writeAll(chunk, 0, chunk.byteLength, position, signal), + signal)); }, writev(chunks, options = kNullPrototo) { @@ -983,7 +1005,8 @@ if (getOptionValue('--experimental-stream-iter')) { if (bytesRemaining > 0) bytesRemaining -= totalSize; const position = pos; if (pos >= 0) pos += totalSize; - return trackOperation(writevAll(chunks, position, signal)); + return trackOperation(enqueueWrite( + () => writevAll(chunks, position, signal), signal)); }, writeSync(chunk) { diff --git a/test/parallel/test-fs-promises-file-handle-writer.js b/test/parallel/test-fs-promises-file-handle-writer.js index a636844f8e57..1074c8156aad 100644 --- a/test/parallel/test-fs-promises-file-handle-writer.js +++ b/test/parallel/test-fs-promises-file-handle-writer.js @@ -1133,6 +1133,69 @@ async function testWriterWebIDLConversion() { // Run all tests // ============================================================================= +// ============================================================================= +// Overlapping (un-awaited) writes must land in call order +// ============================================================================= + +function makeTestData(size) { + const data = Buffer.allocUnsafe(size); + let seed = 0x9e3779b9; + for (let i = 0; i < size; i++) { + seed ^= seed << 13; seed ^= seed >>> 17; seed ^= seed << 5; + data[i] = seed & 0xff; + } + return data; +} + +async function testConcurrentWritesPreserveOrder() { + const data = makeTestData(512 * 1024); + for (const options of [{}, { start: 0 }]) { + for (const useWritev of [false, true]) { + const filePath = path.join( + tmpDir, + `writer-concurrent-${options.start ?? 'none'}-${useWritev}.bin`); + const fh = await open(filePath, 'w'); + const w = fh.writer(options); + const writes = []; + let offset = 0; + let i = 0; + while (offset < data.length) { + const size = 1 + ((i++ * 7919) % 9000); + const end = Math.min(offset + size, data.length); + const chunk = data.subarray(offset, end); + writes.push(useWritev ? + w.writev([chunk.subarray(0, 1), chunk.subarray(1)]) : + w.write(chunk)); + offset = end; + } + await Promise.all(writes); + assert.strictEqual(await w.end(), data.length); + await fh.close(); + assert.deepStrictEqual(fs.readFileSync(filePath), data); + } + } +} + +async function testFailStopsQueuedWrites() { + const filePath = path.join(tmpDir, 'writer-fail-queued.bin'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + const reason = new Error('stop'); + const writes = []; + for (let i = 0; i < 10; i++) { + writes.push(w.write(Buffer.alloc(1024, i))); + } + w.fail(reason); + const results = await Promise.allSettled(writes); + for (const result of results) { + assert.strictEqual(result.status, 'rejected'); + assert.strictEqual(result.reason, reason); + } + await fh.close(); + // None of the queued writes may reach the file after fail(). + assert.strictEqual(fs.statSync(filePath).size, 0); +} + Promise.all([ testBasicWrite(), testBasicWritev(), @@ -1187,4 +1250,6 @@ Promise.all([ testWriterLimitAndStart(), testWriterArgumentValidation(), testWriterWebIDLConversion(), + testConcurrentWritesPreserveOrder(), + testFailStopsQueuedWrites(), ]).then(common.mustCall()); From 69148ee0ffbc80d25512d4e226c13cf0be96c2c3 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:39:48 +0000 Subject: [PATCH 65/98] fs: lock FileHandle on first read in pull() and pullSync() pull() and pullSync() locked the handle as soon as they were called, but only released the lock from inside the iteration. An iterable that was created but never consumed therefore left the handle locked forever, so every later pull(), pullSync() and writer() call failed with ERR_INVALID_STATE. pullSync() also took a reference on the handle eagerly, and returning an iterator that had not started unlocked the handle even if another consumer held the lock. Take the lock (and the reference) when iteration actually starts, as the documentation already describes ("locked while the iterable is being consumed"). Iterating after the handle has been closed now fails with ERR_INVALID_STATE instead of reading from a stale descriptor. testPullLocking is updated accordingly: a second iterable may be created while the first is unconsumed, but consuming it while the first is being consumed still fails. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/fs/promises.js | 43 ++++++++++----- .../test-fs-promises-file-handle-pull.js | 53 +++++++++++++++++-- .../test-fs-promises-file-handle-pullsync.js | 26 +++++++++ 3 files changed, 106 insertions(+), 16 deletions(-) diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 9bd430df795e..aaafe4533d61 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -480,6 +480,18 @@ if (getOptionValue('--experimental-stream-iter')) { const kNullPrototo = { __proto__: null }; const kDefaultChunkSize = 131072; const kNone = -1; + + // Called when iteration of a pull()/pullSync() iterable actually starts. + function lockForIteration(handle, fd) { + if (handle[kFd] === kNone || handle[kFd] !== fd) + throw new ERR_INVALID_STATE('The FileHandle is closed'); + if (handle[kClosePromise]) + throw new ERR_INVALID_STATE('The FileHandle is closing'); + if (handle[kLocked]) + throw new ERR_INVALID_STATE('The FileHandle is locked'); + handle[kLocked] = true; + } + /** * Return the file contents as an AsyncIterable using the * new streams pull model. Optional transforms and options (including @@ -527,11 +539,13 @@ if (getOptionValue('--experimental-stream-iter')) { validateAbortSignal(signal, 'options.signal'); } - this[kLocked] = true; - const source = { __proto__: null, async *[SymbolAsyncIterator]() { + // The handle is locked only while the iterable is actually being + // consumed. Locking eagerly in pull() would leave the handle locked + // forever if the returned iterable were never iterated. + lockForIteration(handle, fd); handle[kRef](); try { if (signal) { @@ -641,10 +655,6 @@ if (getOptionValue('--experimental-stream-iter')) { validateInteger(readSize, 'options.chunkSize', 1); } - this[kLocked] = true; - - handle[kRef](); - function cleanup() { handle[kLocked] = false; handle[kUnref](); @@ -656,15 +666,24 @@ if (getOptionValue('--experimental-stream-iter')) { const source = { __proto__: null, [SymbolIterator]() { + let started = false; let done = false; return { __proto__: null, next() { - if (done || remaining === 0) { - if (!done) { - done = true; - cleanup(); - } + if (done) { + return { done: true, value: undefined }; + } + if (!started) { + // Lock lazily, on the first read, so that an iterable that is + // never consumed does not leave the handle locked forever. + lockForIteration(handle, fd); + handle[kRef](); + started = true; + } + if (remaining === 0) { + done = true; + cleanup(); return { done: true, value: undefined }; } const toRead = remaining > 0 ? @@ -692,7 +711,7 @@ if (getOptionValue('--experimental-stream-iter')) { return() { if (!done) { done = true; - cleanup(); + if (started) cleanup(); } return { done: true, value: undefined }; }, diff --git a/test/parallel/test-fs-promises-file-handle-pull.js b/test/parallel/test-fs-promises-file-handle-pull.js index cdfaf273f933..055a1e395298 100644 --- a/test/parallel/test-fs-promises-file-handle-pull.js +++ b/test/parallel/test-fs-promises-file-handle-pull.js @@ -148,17 +148,30 @@ async function testPullLocking() { const fh = await open(filePath, 'r'); try { - // First pull locks the handle + // The handle is locked once the first iterable starts being consumed. const readable = fh.pull(); + const other = fh.pull(); + const iter = readable[Symbol.asyncIterator](); + const first = await iter.next(); + assert.strictEqual(first.done, false); - // Second pull while locked should throw + // Consuming a second iterable while locked should fail, and so should + // creating new consumers. + await assert.rejects( + other[Symbol.asyncIterator]().next(), + { code: 'ERR_INVALID_STATE' }, + ); assert.throws( () => fh.pull(), { code: 'ERR_INVALID_STATE' }, ); + assert.throws( + () => fh.writer(), + { code: 'ERR_INVALID_STATE' }, + ); - // Consume the first stream to unlock - await text(readable); + // Finish consuming the first stream to unlock + while (!(await iter.next()).done); // Now it should be usable again const readable2 = fh.pull(); @@ -417,6 +430,36 @@ async function testPullSyncArgumentValidation() { } } +// ============================================================================= +// An iterable that is never consumed must not lock the handle +// ============================================================================= + +async function testPullUnconsumedDoesNotLock() { + const filePath = path.join(tmpDir, 'pull-unconsumed.txt'); + fs.writeFileSync(filePath, 'unconsumed'); + + const fh = await open(filePath, 'r'); + try { + fh.pull(); + fh.pull((chunks) => chunks); + assert.strictEqual(await text(fh.pull()), 'unconsumed'); + const w = fh.writer(); + assert.strictEqual(w.endSync(), 0); + } finally { + await fh.close(); + } +} + +async function testPullAfterCloseRejectsOnIteration() { + const filePath = path.join(tmpDir, 'pull-closed-before-iter.txt'); + fs.writeFileSync(filePath, 'data'); + + const fh = await open(filePath, 'r'); + const readable = fh.pull(); + await fh.close(); + await assert.rejects(text(readable), { code: 'ERR_INVALID_STATE' }); +} + Promise.all([ testBasicPull(), testPullBinary(), @@ -437,4 +480,6 @@ Promise.all([ testPullChunkSize(), testPullChunkSizeSmall(), testPullSyncArgumentValidation(), + testPullUnconsumedDoesNotLock(), + testPullAfterCloseRejectsOnIteration(), ]).then(common.mustCall()); diff --git a/test/parallel/test-fs-promises-file-handle-pullsync.js b/test/parallel/test-fs-promises-file-handle-pullsync.js index 20c429972573..82bb5d82393f 100644 --- a/test/parallel/test-fs-promises-file-handle-pullsync.js +++ b/test/parallel/test-fs-promises-file-handle-pullsync.js @@ -473,6 +473,31 @@ async function testPullArgumentValidation() { // Run all tests // ============================================================================= +// ============================================================================= +// An iterable that is never consumed must not lock the handle +// ============================================================================= + +async function testPullSyncUnconsumedDoesNotLock() { + const filePath = path.join(tmpDir, 'pullsync-unconsumed.txt'); + fs.writeFileSync(filePath, 'unconsumed'); + + const fh = await open(filePath, 'r'); + try { + fh.pullSync(); + // Returning an iterator that never started must not unlock or close + // the handle on behalf of another consumer. + fh.pullSync()[Symbol.iterator]().return(); + const iter = fh.pullSync()[Symbol.iterator](); + assert.strictEqual(iter.next().done, false); + assert.throws(() => fh.writer(), { code: 'ERR_INVALID_STATE' }); + iter.return(); + const w = fh.writer(); + assert.strictEqual(w.endSync(), 0); + } finally { + await fh.close(); + } +} + Promise.all([ testBasicPullSync(), testLargeFile(), @@ -495,4 +520,5 @@ Promise.all([ testPullSyncChunkSize(), testWriterChunkSize(), testPullArgumentValidation(), + testPullSyncUnconsumedDoesNotLock(), ]).then(common.mustCall()); From fa49de9ba67edb443ea10f9824df5c43ca544a7d Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:41:39 +0000 Subject: [PATCH 66/98] stream: keep share buffer when all consumers detach When the last consumer of a share() or shareSync() detached, the min cursor fell back to the end of the buffer, so every buffered entry was trimmed, including entries that the detaching consumer had not read. The source had already produced that data, so consumers that attached later silently skipped it. Keep the buffer while there are no consumers, as broadcast() already does, so late-joining consumers start at the oldest entry still in the buffer. Document that the source stays open until the share is cancelled or disposed. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/stream_iter.md | 6 +++++ lib/internal/streams/iter/share.js | 10 +++++++++ test/parallel/test-stream-iter-share-async.js | 22 +++++++++++++++++++ test/parallel/test-stream-iter-share-sync.js | 21 ++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index be00bbd510bb..6728aeb033df 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -1440,6 +1440,12 @@ Create a pull-model multi-consumer shared stream. Unlike `broadcast()`, the source is only read when a consumer pulls. Multiple consumers share a single buffer. +A consumer created with `share.pull()` starts reading at the oldest entry still +in the buffer. Entries are released once every consumer has read them. When +every consumer has detached, the buffered data is kept for consumers that +attach later, and the source is not closed. Call `share.cancel()` (or dispose +the share) to release the source once it is no longer needed. + ```mjs import { from, share, text } from 'node:stream/iter'; diff --git a/lib/internal/streams/iter/share.js b/lib/internal/streams/iter/share.js index b53da4608b23..699feeb51c54 100644 --- a/lib/internal/streams/iter/share.js +++ b/lib/internal/streams/iter/share.js @@ -448,6 +448,11 @@ class ShareImpl { } #tryTrimBuffer() { + // Retain buffered data for consumers that attach while none are active. + // Without this, the last consumer detaching would discard entries that + // a late-joining consumer has not read and cannot get back from the + // source. + if (this.#consumers.size === 0) return; if (this.#cachedMinCursorConsumers === 0) { this.#recomputeMinCursor(); } @@ -744,6 +749,11 @@ class SyncShareImpl { } #tryTrimBuffer() { + // Retain buffered data for consumers that attach while none are active. + // Without this, the last consumer detaching would discard entries that + // a late-joining consumer has not read and cannot get back from the + // source. + if (this.#consumers.size === 0) return; if (this.#cachedMinCursorConsumers === 0) { this.#recomputeMinCursor(); } diff --git a/test/parallel/test-stream-iter-share-async.js b/test/parallel/test-stream-iter-share-async.js index 08fcd0bd338f..29de8e8a141d 100644 --- a/test/parallel/test-stream-iter-share-async.js +++ b/test/parallel/test-stream-iter-share-async.js @@ -379,6 +379,27 @@ async function testShareLateJoiningConsumer() { assert.strictEqual(data2, ''); } +async function testShareRetainsBufferWhenAllConsumersDetach() { + // Data that a consumer had not read yet must stay available to consumers + // that attach after every previous consumer has detached. + const enc = new TextEncoder(); + async function* gen() { + yield [enc.encode('a')]; + yield [enc.encode('b')]; + yield [enc.encode('c')]; + } + const shared = share(gen(), { budget: 16384 }); + const c1 = shared.pull()[Symbol.asyncIterator](); + const c2 = shared.pull()[Symbol.asyncIterator](); + assert.deepStrictEqual((await c1.next()).value, [enc.encode('a')]); + // c2 has not read 'a' yet; it is the last consumer to detach. + await c1.return(); + await c2.return(); + assert.strictEqual(shared.consumerCount, 0); + + assert.strictEqual(await text(shared.pull()), 'abc'); +} + async function testShareConsumerBreak() { // Verify that a consumer breaking mid-iteration detaches properly const enc = new TextEncoder(); @@ -473,6 +494,7 @@ Promise.all([ testShareSourceError(), testShareSourceErrorFollowsBufferedData(), testShareLateJoiningConsumer(), + testShareRetainsBufferWhenAllConsumersDetach(), testShareConsumerBreak(), testShareMultipleConsumersConcurrentPull(), testShareConsumerConcurrentNextCalls(), diff --git a/test/parallel/test-stream-iter-share-sync.js b/test/parallel/test-stream-iter-share-sync.js index fd0184df9bce..6c79a03633a6 100644 --- a/test/parallel/test-stream-iter-share-sync.js +++ b/test/parallel/test-stream-iter-share-sync.js @@ -212,6 +212,26 @@ function testShareSyncDropNewestUnboundedSource() { } // shareSync() accepts string source directly (normalized via fromSync()) +function testShareSyncRetainsBufferWhenAllConsumersDetach() { + // Data that a consumer had not read yet must stay available to consumers + // that attach after every previous consumer has detached. + const enc = new TextEncoder(); + function* gen() { + yield [enc.encode('a')]; + yield [enc.encode('b')]; + yield [enc.encode('c')]; + } + const shared = shareSync(gen(), { budget: 16384 }); + const c1 = shared.pull()[Symbol.iterator](); + const c2 = shared.pull()[Symbol.iterator](); + assert.deepStrictEqual(c1.next().value, [enc.encode('a')]); + c1.return(); + c2.return(); + assert.strictEqual(shared.consumerCount, 0); + + assert.strictEqual(textSync(shared.pull()), 'abc'); +} + function testShareSyncStringSource() { const shared = shareSync('hello-sync-share'); const result = textSync(shared.pull()); @@ -230,4 +250,5 @@ Promise.all([ testShareSyncDropNewest(), testShareSyncDropNewestUnboundedSource(), testShareSyncStringSource(), + testShareSyncRetainsBufferWhenAllConsumersDetach(), ]).then(common.mustCall()); From 42ae08594e431409ad5e8aad8b08eac3de2b83dd Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:42:45 +0000 Subject: [PATCH 67/98] stream: detach shareSync consumers on strict budget errors When a shareSync() consumer needed to pull while the buffer was at or above the budget under the 'strict' policy, it threw ERR_OUT_OF_RANGE but stayed registered. Neither for...of nor a pullSync() transform pipeline calls return() when next() throws, so the abandoned consumer kept its cursor forever, pinned every entry pulled afterwards, and made the remaining consumers fail with ERR_OUT_OF_RANGE as well. Detach the consumer before throwing, as the async share() already does (see 1ad67bc66b2), and document the behavior for both. The spec notes that a rejected strict pull does not terminate the consumer's iterator so that it may be retried. That is not safe with the common iteration patterns, and will be raised with the spec editors. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/stream_iter.md | 7 +++ lib/internal/streams/iter/share.js | 14 +++++- test/parallel/test-stream-iter-share-sync.js | 52 ++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index 6728aeb033df..edb2e17b5df9 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -1446,6 +1446,13 @@ every consumer has detached, the buffered data is kept for consumers that attach later, and the source is not closed. Call `share.cancel()` (or dispose the share) to release the source once it is no longer needed. +With `'strict'` backpressure, a consumer that needs to pull from the source +while the buffer is at or above `budget` is rejected with `ERR_OUT_OF_RANGE` +and detached; further reads from that consumer complete with `{ done: true }`. +Detaching keeps a consumer that is not retried (for example, one read with +`for await...of`, which does not call `return()` when a read rejects) from +holding buffered data and blocking the other consumers. + ```mjs import { from, share, text } from 'node:stream/iter'; diff --git a/lib/internal/streams/iter/share.js b/lib/internal/streams/iter/share.js index 699feeb51c54..7c5246551524 100644 --- a/lib/internal/streams/iter/share.js +++ b/lib/internal/streams/iter/share.js @@ -615,10 +615,20 @@ class SyncShareImpl { let dropped = false; if (self.#bufferedBytes >= self.#options.budget) { switch (self.#options.backpressure) { - case 'strict': - throw new ERR_OUT_OF_RANGE( + case 'strict': { + const error = new ERR_OUT_OF_RANGE( 'buffered bytes', `< ${self.#options.budget}`, self.#bufferedBytes); + // Detach before throwing, as the async share does. Neither + // for...of nor a transform pipeline calls return() when + // next() throws, so a consumer left registered here would + // keep its cursor forever and wedge the other consumers. + state.detached = true; + if (self.#deleteConsumer(state)) { + self.#tryTrimBuffer(); + } + throw error; + } case 'drop-oldest': while (self.#bufferedBytes >= self.#options.budget && self.#buffer.length > 0) { diff --git a/test/parallel/test-stream-iter-share-sync.js b/test/parallel/test-stream-iter-share-sync.js index 6c79a03633a6..2d70ea45456f 100644 --- a/test/parallel/test-stream-iter-share-sync.js +++ b/test/parallel/test-stream-iter-share-sync.js @@ -232,6 +232,56 @@ function testShareSyncRetainsBufferWhenAllConsumersDetach() { assert.strictEqual(textSync(shared.pull()), 'abc'); } +function testShareSyncStrictBackpressureDetaches() { + for (const transformed of [false, true]) { + function* source() { + for (let i = 0; i < 10; i++) { + yield [new Uint8Array(16384)]; + } + } + const shared = shareSync(source(), { + budget: 32768, + backpressure: 'strict', + }); + const consumer = transformed ? + shared.pull((chunks) => chunks) : shared.pull(); + const fast = consumer[Symbol.iterator](); + // This consumer prevents the buffer from being trimmed. + const slow = shared.pull()[Symbol.iterator](); + + fast.next(); + fast.next(); + assert.throws(() => fast.next(), { code: 'ERR_OUT_OF_RANGE' }); + // The rejected consumer is detached, as with the async share. + assert.strictEqual(shared.consumerCount, 1); + assert.strictEqual(fast.next().done, true); + + // The detached consumer no longer pins the buffer, so the remaining + // consumer can read the whole source. + let count = 0; + while (!slow.next().done) count++; + assert.strictEqual(count, 10); + } +} + +function testShareSyncStrictForOfDoesNotWedgeOthers() { + // for...of does not call return() when next() throws. The consumer that + // hit the budget must still not keep the other consumers from reading. + function* source() { + for (let i = 0; i < 20; i++) yield [new Uint8Array(8192)]; + } + const shared = shareSync(source(), { budget: 16384, backpressure: 'strict' }); + const slow = shared.pull()[Symbol.iterator](); + assert.throws(() => { + // eslint-disable-next-line no-unused-vars + for (const _ of shared.pull()) { /* consume */ } + }, { code: 'ERR_OUT_OF_RANGE' }); + assert.strictEqual(shared.consumerCount, 1); + let count = 0; + while (!slow.next().done) count++; + assert.strictEqual(count, 20); +} + function testShareSyncStringSource() { const shared = shareSync('hello-sync-share'); const result = textSync(shared.pull()); @@ -251,4 +301,6 @@ Promise.all([ testShareSyncDropNewestUnboundedSource(), testShareSyncStringSource(), testShareSyncRetainsBufferWhenAllConsumersDetach(), + testShareSyncStrictBackpressureDetaches(), + testShareSyncStrictForOfDoesNotWedgeOthers(), ]).then(common.mustCall()); From 30cbc4e643fc53911b4454497fe8ef6f754d9193 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:44:05 +0000 Subject: [PATCH 68/98] stream: reject drop-newest backpressure in shareSync() With 'drop-newest', a shareSync() consumer that needed to pull while the buffer was at the budget discarded one entry from the source and then returned { done: true } even though the source was not exhausted. for...of loops, and anything else that trusts the iterator protocol, silently stopped consuming. There is no correct alternative in a synchronous context: the slowest consumer cannot advance while another consumer's next() is running, so the call can neither wait for budget nor keep discarding until budget is released. Reject 'drop-newest' in shareSync() with ERR_INVALID_ARG_VALUE, as is already done for 'unbounded'. The two tests that asserted the previous "done but not detached" behavior are replaced by one that checks the rejection. Also document how 'unbounded' and 'drop-newest' make the async share() wait for the slowest consumer. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/stream_iter.md | 19 +++-- lib/internal/streams/iter/share.js | 29 +++----- test/parallel/test-stream-iter-share-sync.js | 74 +++----------------- 3 files changed, 32 insertions(+), 90 deletions(-) diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index edb2e17b5df9..77eb79a20511 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -1453,6 +1453,13 @@ Detaching keeps a consumer that is not retried (for example, one read with `for await...of`, which does not call `return()` when a read rejects) from holding buffered data and blocking the other consumers. +With `'unbounded'`, such a consumer waits until the slowest consumer releases +budget. With `'drop-newest'`, the entry pulled from the source is discarded +and the consumer then waits in the same way, so in both cases a stalled +consumer also stalls the consumers that are ahead of it. Only `'drop-oldest'` +lets consumers that are ahead continue, by discarding the oldest buffered +entries that the slowest consumer has not read yet. + ```mjs import { from, share, text } from 'node:stream/iter'; @@ -1548,18 +1555,16 @@ added: v25.9.0 * `options` {Object} * `budget` {number} Must be >= 16384. **Default:** `65536`. - * `backpressure` {string} `'strict'`, `'drop-oldest'`, or `'drop-newest'`. + * `backpressure` {string} `'strict'` or `'drop-oldest'`. **Default:** `'strict'`. * Returns: {SyncShare} Synchronous version of [`share()`][]. -Because there is no way to wait in a synchronous context, `'unbounded'` is not -supported and throws `ERR_INVALID_ARG_VALUE`. With `'drop-newest'`, a consumer -that reaches the end of the buffer while the budget is exhausted discards a -single entry from the source and then returns `{ done: true }` without a -value; the consumer is not detached, so it can resume once the slowest -consumer advances and releases budget. +A synchronous consumer cannot wait for the slowest consumer to release budget, +and the slowest consumer cannot advance while another consumer's read is +running. `'unbounded'` and `'drop-newest'` are therefore not supported and +throw `ERR_INVALID_ARG_VALUE`. ### Class: `SyncShare` diff --git a/lib/internal/streams/iter/share.js b/lib/internal/streams/iter/share.js index 7c5246551524..5439dfeb3d66 100644 --- a/lib/internal/streams/iter/share.js +++ b/lib/internal/streams/iter/share.js @@ -611,8 +611,8 @@ class SyncShareImpl { return { __proto__: null, done: true, value: undefined }; } - // Check buffer limit - let dropped = false; + // Check buffer limit. 'unbounded' and 'drop-newest' are rejected + // by shareSync(). if (self.#bufferedBytes >= self.#options.budget) { switch (self.#options.backpressure) { case 'strict': { @@ -644,21 +644,10 @@ class SyncShareImpl { } self.#recomputeMinCursor(); break; - case 'drop-newest': - // Discarding does not reclaim budget, and the slowest - // consumer cannot advance while this synchronous next() is - // running, so at most one entry may be dropped per call. - // Looping here would spin forever on an unbounded source - // and drain a finite one in a single call. - self.#pullFromSource(true); - dropped = true; - break; } } - if (!dropped) { - self.#pullFromSource(); - } + self.#pullFromSource(); if (self.#sourceError !== kNoShareError) { state.detached = true; @@ -737,7 +726,7 @@ class SyncShareImpl { this.cancel(); } - #pullFromSource(discard = false) { + #pullFromSource() { if (this.#sourceExhausted || this.#cancelled) return; try { @@ -747,7 +736,7 @@ class SyncShareImpl { if (result.done) { this.#sourceExhausted = true; - } else if (!discard) { + } else { const entry = createBatchEntry(result.value); this.#buffer.push(entry); this.#bufferedBytes += entry.byteLength; @@ -858,10 +847,14 @@ function shareSync(source, options = { __proto__: null }) { backpressure = 'strict', } = options; validateInteger(budget, 'options.budget', 16384); - if (backpressure === 'unbounded') { + // A synchronous consumer can neither wait for the slowest consumer to + // release budget ('unbounded') nor keep pulling and discarding until it + // does ('drop-newest'): the slowest consumer cannot advance while the + // call is running. + if (backpressure === 'unbounded' || backpressure === 'drop-newest') { throw new ERR_INVALID_ARG_VALUE( 'options.backpressure', backpressure, - 'unbounded is not supported by shareSync()'); + `${backpressure} is not supported by shareSync()`); } const opts = { diff --git a/test/parallel/test-stream-iter-share-sync.js b/test/parallel/test-stream-iter-share-sync.js index 2d70ea45456f..2721905bc936 100644 --- a/test/parallel/test-stream-iter-share-sync.js +++ b/test/parallel/test-stream-iter-share-sync.js @@ -146,69 +146,14 @@ function testShareSyncRejectsUnbounded() { ); } -function testShareSyncDropNewest() { - let pulls = 0; - function* source() { - for (let i = 0; i < 4; i++) { - pulls++; - const chunk = new Uint8Array(16384); - chunk[0] = i; - yield [chunk]; - } - } - - const shared = shareSync(source(), { - budget: 16384, - backpressure: 'drop-newest', - }); - const fast = shared.pull()[Symbol.iterator](); - const slow = shared.pull()[Symbol.iterator](); - - assert.strictEqual(fast.next().value[0][0], 0); - - // The budget is exhausted and the slow consumer cannot advance while this - // call is running, so exactly one entry is dropped and no value is - // available. The consumer is not detached. - assert.strictEqual(fast.next().done, true); - assert.strictEqual(pulls, 2); - - // The slow consumer still sees the buffered entry, which releases budget. - assert.strictEqual(slow.next().value[0][0], 0); - - // Entry 1 was dropped for every consumer, so both resume at entry 2. - assert.strictEqual(slow.next().value[0][0], 2); - assert.strictEqual(pulls, 3); - assert.strictEqual(fast.next().value[0][0], 2); -} - -// Regression test: a full buffer must not spin pulling-and-discarding from an -// unbounded source, since discarding never reclaims budget. -function testShareSyncDropNewestUnboundedSource() { - let pulls = 0; - function* source() { - for (;;) { - pulls++; - yield [new Uint8Array(16384)]; - } - } - - const shared = shareSync(source(), { - budget: 16384, - backpressure: 'drop-newest', - }); - const fast = shared.pull()[Symbol.iterator](); - shared.pull(); - - assert.strictEqual(fast.next().done, false); - assert.strictEqual(pulls, 1); - - // Each blocked call drops at most one entry and returns without a value. - for (let i = 0; i < 3; i++) { - assert.strictEqual(fast.next().done, true); - assert.strictEqual(pulls, 2 + i); - } - - shared.cancel(); +function testShareSyncRejectsDropNewest() { + // A synchronous consumer can neither wait for the slowest consumer nor keep + // discarding until it advances, so 'drop-newest' is rejected like + // 'unbounded'. + assert.throws( + () => shareSync(fromSync('data'), { backpressure: 'drop-newest' }), + { code: 'ERR_INVALID_ARG_VALUE' }, + ); } // shareSync() accepts string source directly (normalized via fromSync()) @@ -297,8 +242,7 @@ Promise.all([ testShareSyncCancelWithFalsyReason(), testShareSyncSourceError(), testShareSyncRejectsUnbounded(), - testShareSyncDropNewest(), - testShareSyncDropNewestUnboundedSource(), + testShareSyncRejectsDropNewest(), testShareSyncStringSource(), testShareSyncRetainsBufferWhenAllConsumersDetach(), testShareSyncStrictBackpressureDetaches(), From 8a47365425dfbd96009883f2c275a027bbbb3b1b Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:45:55 +0000 Subject: [PATCH 69/98] stream: split oversized share batches under drop-oldest share() and shareSync() buffer each batch pulled from the source as a single entry, and the 'drop-oldest' policy evicts whole entries until the buffer is below the budget. from() and fromSync() combine up to 128 values of a sync source into one batch, so a single entry can be many times larger than the budget. Evicting it discarded every chunk in it, including chunks that slower consumers had not read yet: a consumer could lose the entire stream while a faster consumer read all of it. When the policy is 'drop-oldest', split batches that are larger than the budget into consecutive entries that are each smaller than it, so eviction keeps the newest chunks that fit within the budget. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/stream_iter.md | 4 +- lib/internal/streams/iter/share.js | 47 ++++++++++++++++--- lib/internal/streams/iter/utils.js | 31 ++++++++++++ test/parallel/test-stream-iter-share-async.js | 38 +++++++++++++++ test/parallel/test-stream-iter-share-sync.js | 38 +++++++++++++++ 5 files changed, 151 insertions(+), 7 deletions(-) diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index 77eb79a20511..deb6a11bb85b 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -1458,7 +1458,9 @@ budget. With `'drop-newest'`, the entry pulled from the source is discarded and the consumer then waits in the same way, so in both cases a stalled consumer also stalls the consumers that are ahead of it. Only `'drop-oldest'` lets consumers that are ahead continue, by discarding the oldest buffered -entries that the slowest consumer has not read yet. +entries that the slowest consumer has not read yet. A batch pulled from the +source that is larger than `budget` is split into smaller entries first, so +eviction keeps the newest chunks that fit within the budget. ```mjs import { from, share, text } from 'node:stream/iter'; diff --git a/lib/internal/streams/iter/share.js b/lib/internal/streams/iter/share.js index 5439dfeb3d66..ac287d9472a2 100644 --- a/lib/internal/streams/iter/share.js +++ b/lib/internal/streams/iter/share.js @@ -44,6 +44,7 @@ const { getMinCursor, onSignalAbort, parsePullArgs, + splitBatchEntry, validateBatchEntry, } = require('internal/streams/iter/utils'); const { @@ -427,9 +428,7 @@ class ShareImpl { if (result.done) { this.#sourceExhausted = true; } else if (!discard) { - const entry = createBatchEntry(result.value); - this.#buffer.push(entry); - this.#bufferedBytes += entry.byteLength; + this.#bufferBatch(result.value); } } catch (error) { this.#sourceError = error; @@ -447,6 +446,25 @@ class ShareImpl { })(); } + #bufferBatch(batch) { + const entry = createBatchEntry(batch); + // 'drop-oldest' evicts whole entries. A single pulled batch can be much + // larger than the budget (for example when from() combines many values + // of a sync source), and evicting it would discard every chunk in it, + // including ones that no consumer has read. Split such batches so that + // eviction keeps the newest chunks that fit the budget. + const entries = this.#options.backpressure === 'drop-oldest' ? + splitBatchEntry(entry, this.#options.budget) : undefined; + if (entries === undefined) { + this.#buffer.push(entry); + } else { + for (let i = 0; i < entries.length; i++) { + this.#buffer.push(entries[i]); + } + } + this.#bufferedBytes += entry.byteLength; + } + #tryTrimBuffer() { // Retain buffered data for consumers that attach while none are active. // Without this, the last consumer detaching would discard entries that @@ -737,9 +755,7 @@ class SyncShareImpl { if (result.done) { this.#sourceExhausted = true; } else { - const entry = createBatchEntry(result.value); - this.#buffer.push(entry); - this.#bufferedBytes += entry.byteLength; + this.#bufferBatch(result.value); } } catch (error) { this.#sourceError = error; @@ -747,6 +763,25 @@ class SyncShareImpl { } } + #bufferBatch(batch) { + const entry = createBatchEntry(batch); + // 'drop-oldest' evicts whole entries. A single pulled batch can be much + // larger than the budget (for example when from() combines many values + // of a sync source), and evicting it would discard every chunk in it, + // including ones that no consumer has read. Split such batches so that + // eviction keeps the newest chunks that fit the budget. + const entries = this.#options.backpressure === 'drop-oldest' ? + splitBatchEntry(entry, this.#options.budget) : undefined; + if (entries === undefined) { + this.#buffer.push(entry); + } else { + for (let i = 0; i < entries.length; i++) { + this.#buffer.push(entries[i]); + } + } + this.#bufferedBytes += entry.byteLength; + } + #tryTrimBuffer() { // Retain buffered data for consumers that attach while none are active. // Without this, the last consumer detaching would discard entries that diff --git a/lib/internal/streams/iter/utils.js b/lib/internal/streams/iter/utils.js index a6069c9ed3b7..34698dbdf0da 100644 --- a/lib/internal/streams/iter/utils.js +++ b/lib/internal/streams/iter/utils.js @@ -4,6 +4,7 @@ const { Array, ArrayBufferPrototypeGetByteLength, ArrayBufferPrototypeGetDetached, + ArrayPrototypePush, ArrayPrototypeSlice, PromiseResolve, PromiseWithResolvers, @@ -245,6 +246,35 @@ function createBatchEntry(chunks) { return { __proto__: null, views, byteLength }; } +/** + * Split a batch entry into consecutive entries that are each smaller than + * `limit` bytes, preserving chunk order. A single chunk of `limit` bytes or + * more forms an entry on its own. Returns `undefined` if no split is needed. + * @param {{ views: object[], byteLength: number }} entry + * @param {number} limit + * @returns {Array<{ views: object[], byteLength: number }>|undefined} + */ +function splitBatchEntry(entry, limit) { + const { views } = entry; + if (entry.byteLength < limit || views.length < 2) return undefined; + const entries = []; + let current = []; + let byteLength = 0; + for (let i = 0; i < views.length; i++) { + const view = views[i]; + if (current.length > 0 && byteLength + view.byteLength >= limit) { + ArrayPrototypePush(entries, + { __proto__: null, views: current, byteLength }); + current = []; + byteLength = 0; + } + ArrayPrototypePush(current, view); + byteLength += view.byteLength; + } + ArrayPrototypePush(entries, { __proto__: null, views: current, byteLength }); + return entries; +} + function validateBatchEntry(entry) { const chunks = new Array(entry.views.length); for (let i = 0; i < entry.views.length; i++) { @@ -456,6 +486,7 @@ module.exports = { concatBytes, convertChunks, createBatchEntry, + splitBatchEntry, getProtocolMethod, getWriterSignal, getMinCursor, diff --git a/test/parallel/test-stream-iter-share-async.js b/test/parallel/test-stream-iter-share-async.js index 29de8e8a141d..8ea06427543e 100644 --- a/test/parallel/test-stream-iter-share-async.js +++ b/test/parallel/test-stream-iter-share-async.js @@ -400,6 +400,43 @@ async function testShareRetainsBufferWhenAllConsumersDetach() { assert.strictEqual(await text(shared.pull()), 'abc'); } +async function testShareDropOldestSplitsOversizedBatches() { + // from() combines the values of this sync generator into a single batch + // that is much larger than the budget. Evicting that batch as a whole would + // leave the slower consumer with nothing at all. + function* source() { + for (let i = 0; i < 50; i++) { + const chunk = new Uint8Array(4096); + chunk[0] = i; + yield chunk; + } + } + const shared = share(source(), { + budget: 65536, + backpressure: 'drop-oldest', + }); + const fast = shared.pull()[Symbol.asyncIterator](); + const slow = shared.pull()[Symbol.asyncIterator](); + + const fastSeen = []; + for (let r = await fast.next(); !r.done; r = await fast.next()) { + for (const chunk of r.value) fastSeen.push(chunk[0]); + } + assert.deepStrictEqual(fastSeen, Array.from({ length: 50 }, (_, i) => i)); + + const slowSeen = []; + for (let r = await slow.next(); !r.done; r = await slow.next()) { + for (const chunk of r.value) slowSeen.push(chunk[0]); + } + // The slow consumer lost the oldest chunks but keeps an in-order suffix + // that fits the budget. + assert.ok(slowSeen.length > 0); + assert.ok(slowSeen.length * 4096 < 65536); + assert.deepStrictEqual( + slowSeen, + Array.from({ length: slowSeen.length }, (_, i) => 50 - slowSeen.length + i)); +} + async function testShareConsumerBreak() { // Verify that a consumer breaking mid-iteration detaches properly const enc = new TextEncoder(); @@ -495,6 +532,7 @@ Promise.all([ testShareSourceErrorFollowsBufferedData(), testShareLateJoiningConsumer(), testShareRetainsBufferWhenAllConsumersDetach(), + testShareDropOldestSplitsOversizedBatches(), testShareConsumerBreak(), testShareMultipleConsumersConcurrentPull(), testShareConsumerConcurrentNextCalls(), diff --git a/test/parallel/test-stream-iter-share-sync.js b/test/parallel/test-stream-iter-share-sync.js index 2721905bc936..dd392d9c9927 100644 --- a/test/parallel/test-stream-iter-share-sync.js +++ b/test/parallel/test-stream-iter-share-sync.js @@ -227,6 +227,43 @@ function testShareSyncStrictForOfDoesNotWedgeOthers() { assert.strictEqual(count, 20); } +function testShareSyncDropOldestSplitsOversizedBatches() { + // fromSync() combines the values of this generator into a single batch + // that is much larger than the budget. Evicting that batch as a whole would + // leave the slower consumer with nothing at all. + function* source() { + for (let i = 0; i < 50; i++) { + const chunk = new Uint8Array(4096); + chunk[0] = i; + yield chunk; + } + } + const shared = shareSync(source(), { + budget: 65536, + backpressure: 'drop-oldest', + }); + const fast = shared.pull()[Symbol.iterator](); + const slow = shared.pull()[Symbol.iterator](); + + const fastSeen = []; + for (let r = fast.next(); !r.done; r = fast.next()) { + for (const chunk of r.value) fastSeen.push(chunk[0]); + } + assert.deepStrictEqual(fastSeen, Array.from({ length: 50 }, (_, i) => i)); + + const slowSeen = []; + for (let r = slow.next(); !r.done; r = slow.next()) { + for (const chunk of r.value) slowSeen.push(chunk[0]); + } + // The slow consumer lost the oldest chunks but keeps an in-order suffix + // that fits the budget. + assert.ok(slowSeen.length > 0); + assert.ok(slowSeen.length * 4096 < 65536); + assert.deepStrictEqual( + slowSeen, + Array.from({ length: slowSeen.length }, (_, i) => 50 - slowSeen.length + i)); +} + function testShareSyncStringSource() { const shared = shareSync('hello-sync-share'); const result = textSync(shared.pull()); @@ -243,6 +280,7 @@ Promise.all([ testShareSyncSourceError(), testShareSyncRejectsUnbounded(), testShareSyncRejectsDropNewest(), + testShareSyncDropOldestSplitsOversizedBatches(), testShareSyncStringSource(), testShareSyncRetainsBufferWhenAllConsumersDetach(), testShareSyncStrictBackpressureDetaches(), From 6b1e7c4cf1bce24618f8b4de6d932ea94c780613 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:47:19 +0000 Subject: [PATCH 70/98] stream: guard shareSync() against re-entrant source reads If a shareSync() source read from a consumer of the same share while producing a value, the nested read re-entered the source iterator. For generators this threw "Generator is already running" from inside the nested read, which recorded that error as the share's source error while the outer read was still in progress, leaving every consumer in an error state. Fail the nested read with ERR_INVALID_STATE before touching the share's state. The source sees the error and may handle it; if it lets it escape, it becomes the source error as with any other source failure. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/streams/iter/share.js | 13 ++++++ test/parallel/test-stream-iter-share-sync.js | 46 ++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/lib/internal/streams/iter/share.js b/lib/internal/streams/iter/share.js index ac287d9472a2..17dabf5da039 100644 --- a/lib/internal/streams/iter/share.js +++ b/lib/internal/streams/iter/share.js @@ -60,6 +60,7 @@ const { ERR_INVALID_ARG_TYPE, ERR_INVALID_ARG_VALUE, ERR_INVALID_RETURN_VALUE, + ERR_INVALID_STATE, ERR_OUT_OF_RANGE, }, } = require('internal/errors'); @@ -548,6 +549,7 @@ class SyncShareImpl { #sourceExhausted = false; #sourceError = kNoShareError; #cancelled = false; + #pulling = false; #cachedMinCursor = 0; #cachedMinCursorConsumers = 0; /** Cumulative byte size of buffered entries */ @@ -747,6 +749,15 @@ class SyncShareImpl { #pullFromSource() { if (this.#sourceExhausted || this.#cancelled) return; + // A source that reads from its own share would re-enter its own + // iterator. Fail that read without touching the share's state; the + // source sees the error and may handle it. + if (this.#pulling) { + throw new ERR_INVALID_STATE( + 'shareSync() source cannot be read while it is producing a value'); + } + + this.#pulling = true; try { this.#sourceIterator ||= this.#source[SymbolIterator](); @@ -760,6 +771,8 @@ class SyncShareImpl { } catch (error) { this.#sourceError = error; this.#sourceExhausted = true; + } finally { + this.#pulling = false; } } diff --git a/test/parallel/test-stream-iter-share-sync.js b/test/parallel/test-stream-iter-share-sync.js index dd392d9c9927..a843e7311dd1 100644 --- a/test/parallel/test-stream-iter-share-sync.js +++ b/test/parallel/test-stream-iter-share-sync.js @@ -264,6 +264,50 @@ function testShareSyncDropOldestSplitsOversizedBatches() { Array.from({ length: slowSeen.length }, (_, i) => 50 - slowSeen.length + i)); } +function testShareSyncReentrantSourceRead() { + const enc = new TextEncoder(); + let sibling; + let reentrantError; + function* source() { + yield [enc.encode('a')]; + try { + sibling.next(); + } catch (err) { + reentrantError = err; + } + yield [enc.encode('b')]; + } + const shared = shareSync(source()); + const c1 = shared.pull()[Symbol.iterator](); + sibling = shared.pull()[Symbol.iterator](); + + assert.deepStrictEqual(c1.next().value, [enc.encode('a')]); + assert.deepStrictEqual(sibling.next().value, [enc.encode('a')]); + // Pulling 'b' runs the source, which tries to read its own share. + assert.deepStrictEqual(c1.next().value, [enc.encode('b')]); + assert.strictEqual(reentrantError?.code, 'ERR_INVALID_STATE'); + // The failed re-entrant read left the share intact. + assert.deepStrictEqual(sibling.next().value, [enc.encode('b')]); + assert.strictEqual(c1.next().done, true); + assert.strictEqual(sibling.next().done, true); +} + +function testShareSyncReentrantSourceReadUncaught() { + let sibling; + function* source() { + yield [new Uint8Array(1)]; + sibling.next(); + } + const shared = shareSync(source()); + const c1 = shared.pull()[Symbol.iterator](); + sibling = shared.pull()[Symbol.iterator](); + c1.next(); + sibling.next(); + // The error escapes the source, so it becomes the share's source error. + assert.throws(() => c1.next(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => sibling.next(), { code: 'ERR_INVALID_STATE' }); +} + function testShareSyncStringSource() { const shared = shareSync('hello-sync-share'); const result = textSync(shared.pull()); @@ -281,6 +325,8 @@ Promise.all([ testShareSyncRejectsUnbounded(), testShareSyncRejectsDropNewest(), testShareSyncDropOldestSplitsOversizedBatches(), + testShareSyncReentrantSourceRead(), + testShareSyncReentrantSourceReadUncaught(), testShareSyncStringSource(), testShareSyncRetainsBufferWhenAllConsumersDetach(), testShareSyncStrictBackpressureDetaches(), From 69d44aedb4b9ef6e77274a7be6bea7c5ea6c1117 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:51:44 +0000 Subject: [PATCH 71/98] stream: bound pre-batched async values in from() When an async source yielded an already-batched Uint8Array[] value, from() passed it through as-is, however large it was. Every other input shape (a sync source, or an array passed to from() or fromSync() directly) splits such batches into batches of at most 128 chunks, which bounds the memory transforms must allocate per batch. Apply the same bound to async sources. Batches within the bound are still passed through without copying. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/streams/iter/from.js | 4 +--- test/parallel/test-stream-iter-from-async.js | 22 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/internal/streams/iter/from.js b/lib/internal/streams/iter/from.js index cac8fd3565b6..841041469fc5 100644 --- a/lib/internal/streams/iter/from.js +++ b/lib/internal/streams/iter/from.js @@ -547,9 +547,7 @@ async function* normalizeAsyncSource(source, context) { for await (const value of iterable) { // Fast path 1: value is already a Uint8Array[] batch if (isUint8ArrayBatch(value)) { - if (value.length > 0) { - yield value; - } + yield* yieldBoundedBatch(value); continue; } // Fast path 2: value is a single Uint8Array (very common) diff --git a/test/parallel/test-stream-iter-from-async.js b/test/parallel/test-stream-iter-from-async.js index 29ecdf425767..2be5a2c55e4e 100644 --- a/test/parallel/test-stream-iter-from-async.js +++ b/test/parallel/test-stream-iter-from-async.js @@ -105,6 +105,27 @@ async function testFromBoundsNestedAsyncIterable() { assert.strictEqual(nestedClosed, true); } +async function testFromBoundsPreBatchedAsyncValues() { + // An async source yielding an already-batched Uint8Array[] larger than the + // batch bound is split, like the same batch from a sync source. + const big = Array.from({ length: 300 }, (_, i) => new Uint8Array([i & 0xff])); + async function* source() { + yield big; + } + const sizes = []; + for await (const batch of from(source())) sizes.push(batch.length); + assert.deepStrictEqual(sizes, [128, 128, 44]); + + // Batches within the bound are still passed through as-is. + const small = [new Uint8Array([1]), new Uint8Array([2])]; + async function* smallSource() { + yield small; + } + for await (const batch of from(smallSource())) { + assert.strictEqual(batch, small); + } +} + async function testFromSyncIterableAsAsync() { // Sync iterable passed to from() should work function* gen() { @@ -464,6 +485,7 @@ Promise.all([ testFromAsyncIteratorResultShapes(), testFromSourceErrorDoesNotWaitForReturn(), testFromBoundsNestedAsyncIterable(), + testFromBoundsPreBatchedAsyncValues(), testFromSyncIterableAsAsync(), testFromSyncIterableAwaitsPromiseValues(), testFromSyncIterableRejectsNestedAsyncIterable(), From 6eaf91353a98cd05660ccade7328ce51c693b302 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:51:59 +0000 Subject: [PATCH 72/98] stream: do not hold back nested async data in from() When an async source yielded a value that needed normalizing, such as a nested async iterable, from() collected the resulting chunks and only yielded them once 128 had accumulated or the value was fully consumed. A slow nested stream therefore delivered nothing until it ended, an endless one with fewer than 128 chunks in flight delivered nothing at all, and the chunks piled up in memory meanwhile. This affected every API built on from(), e.g. when concatenating streams with `async function*() { yield fromReadable(a); yield fromReadable(b); }`. Yield whatever has been collected right before waiting on a promise or on a nested async iterable. Chunks that are produced together are still batched (up to the same bound). testFromBoundsNestedAsyncIterable asserted that the first batch from an endless nested async iterable held exactly 128 chunks; it now checks that the batch is non-empty and bounded, which is what it guards. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/streams/iter/from.js | 53 ++++++++++++++++---- test/parallel/test-stream-iter-from-async.js | 31 +++++++++++- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/lib/internal/streams/iter/from.js b/lib/internal/streams/iter/from.js index 841041469fc5..ab786dc38258 100644 --- a/lib/internal/streams/iter/from.js +++ b/lib/internal/streams/iter/from.js @@ -61,6 +61,11 @@ const { // allocate output for the entire batch at once. const FROM_BATCH_SIZE = 128; const kNormalizationCancelled = Symbol('kNormalizationCancelled'); +// Yielded by normalizeAsyncValue() (only when `emitFlush` is true) right +// before it waits on a promise or on a nested async iterable. Callers that +// batch chunks yield whatever they have collected so far, so that chunks that +// are already available are not held back until the wait completes. +const kFlushBatch = Symbol('kFlushBatch'); function createNormalizationContext() { return { @@ -444,14 +449,15 @@ function yieldNormalizationAbortable(source, context) { * @yields {Uint8Array} */ async function* normalizeAsyncValue( - value, allowNestedAsyncStreamables = true, context) { + value, allowNestedAsyncStreamables = true, context, emitFlush = false) { throwIfNormalizationCancelled(context); // Handle promises first if (isPromise(value)) { + if (emitFlush) yield kFlushBatch; const resolved = await waitForNormalization(value, context); yield* normalizeAsyncValue( - resolved, allowNestedAsyncStreamables, context); + resolved, allowNestedAsyncStreamables, context, emitFlush); return; } @@ -478,13 +484,15 @@ async function* normalizeAsyncValue( if (asyncStreamableMethod !== undefined) { const result = FunctionPrototypeCall(asyncStreamableMethod, value); if (isPromise(result)) { + if (emitFlush) yield kFlushBatch; yield* normalizeAsyncValue( await waitForNormalization(result, context), allowNestedAsyncStreamables, - context); + context, + emitFlush); } else { yield* normalizeAsyncValue( - result, allowNestedAsyncStreamables, context); + result, allowNestedAsyncStreamables, context, emitFlush); } return; } @@ -493,7 +501,8 @@ async function* normalizeAsyncValue( const streamableMethod = getProtocolMethod(value, toStreamable); if (streamableMethod !== undefined) { const result = FunctionPrototypeCall(streamableMethod, value); - yield* normalizeAsyncValue(result, allowNestedAsyncStreamables, context); + yield* normalizeAsyncValue( + result, allowNestedAsyncStreamables, context, emitFlush); return; } @@ -501,7 +510,7 @@ async function* normalizeAsyncValue( if (ArrayIsArray(value)) { for (let i = 0; i < value.length; i++) { yield* normalizeAsyncValue( - value[i], allowNestedAsyncStreamables, context); + value[i], allowNestedAsyncStreamables, context, emitFlush); } return; } @@ -510,8 +519,11 @@ async function* normalizeAsyncValue( // have both) if (isAsyncIterable(value)) { const iterable = yieldNormalizationAbortable(value, context); + if (emitFlush) yield kFlushBatch; for await (const item of iterable) { - yield* normalizeAsyncValue(item, allowNestedAsyncStreamables, context); + yield* normalizeAsyncValue( + item, allowNestedAsyncStreamables, context, emitFlush); + if (emitFlush) yield kFlushBatch; } return; } @@ -519,7 +531,8 @@ async function* normalizeAsyncValue( // Handle sync iterables if (isSyncIterable(value)) { for (const item of value) { - yield* normalizeAsyncValue(item, allowNestedAsyncStreamables, context); + yield* normalizeAsyncValue( + item, allowNestedAsyncStreamables, context, emitFlush); } return; } @@ -555,9 +568,19 @@ async function* normalizeAsyncSource(source, context) { yield [value]; continue; } - // Slow path: normalize the value + // Slow path: normalize the value. Chunks are batched, but whatever has + // been collected is yielded before waiting on async content (such as a + // nested async iterable), so data is not held back until it ends. let batch = []; - for await (const chunk of normalizeAsyncValue(value, true, context)) { + for await (const chunk of + normalizeAsyncValue(value, true, context, true)) { + if (chunk === kFlushBatch) { + if (batch.length > 0) { + yield batch; + batch = []; + } + continue; + } ArrayPrototypePush(batch, chunk); if (batch.length === FROM_BATCH_SIZE) { yield batch; @@ -602,7 +625,15 @@ async function* normalizeAsyncSource(source, context) { batch = []; } let asyncBatch = []; - for await (const chunk of normalizeAsyncValue(value, false, context)) { + for await (const chunk of + normalizeAsyncValue(value, false, context, true)) { + if (chunk === kFlushBatch) { + if (asyncBatch.length > 0) { + yield asyncBatch; + asyncBatch = []; + } + continue; + } ArrayPrototypePush(asyncBatch, chunk); if (asyncBatch.length === FROM_BATCH_SIZE) { yield asyncBatch; diff --git a/test/parallel/test-stream-iter-from-async.js b/test/parallel/test-stream-iter-from-async.js index 2be5a2c55e4e..d04d7df9aad4 100644 --- a/test/parallel/test-stream-iter-from-async.js +++ b/test/parallel/test-stream-iter-from-async.js @@ -99,12 +99,40 @@ async function testFromBoundsNestedAsyncIterable() { const iterator = from(source())[Symbol.asyncIterator](); const first = await iterator.next(); assert.strictEqual(first.done, false); - assert.strictEqual(first.value.length, 128); + assert.ok(first.value.length > 0); + assert.ok(first.value.length <= 128); await iterator.return(); assert.strictEqual(nestedClosed, true); } +async function testFromDoesNotHoldBackNestedAsyncIterable() { + // Chunks from a nested async iterable must be delivered as they become + // available, not held back until the nested iterable produces more data + // or ends. + const { promise: release, resolve } = Promise.withResolvers(); + async function* nested() { + yield new Uint8Array([1]); + await release; + yield new Uint8Array([2]); + } + + async function* source() { + yield nested(); + yield [new Uint8Array([3]), Promise.resolve(new Uint8Array([4]))]; + } + + const iterator = from(source())[Symbol.asyncIterator](); + assert.deepStrictEqual(await iterator.next(), + { done: false, value: [new Uint8Array([1])] }); + resolve(); + const rest = []; + for (let r = await iterator.next(); !r.done; r = await iterator.next()) { + for (const chunk of r.value) rest.push(chunk[0]); + } + assert.deepStrictEqual(rest, [2, 3, 4]); +} + async function testFromBoundsPreBatchedAsyncValues() { // An async source yielding an already-batched Uint8Array[] larger than the // batch bound is split, like the same batch from a sync source. @@ -485,6 +513,7 @@ Promise.all([ testFromAsyncIteratorResultShapes(), testFromSourceErrorDoesNotWaitForReturn(), testFromBoundsNestedAsyncIterable(), + testFromDoesNotHoldBackNestedAsyncIterable(), testFromBoundsPreBatchedAsyncValues(), testFromSyncIterableAsAsync(), testFromSyncIterableAwaitsPromiseValues(), From 995dee6b6bc2b2fba4e308a6683d3561c6468ef3 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:53:30 +0000 Subject: [PATCH 73/98] stream: honor stream/iter protocols on function objects The protocol lookup used by from(), fromSync(), and the Broadcast/Share helpers only considered values with typeof 'object', so a function implementing, e.g., Symbol.for('Stream.toStreamable') was rejected with ERR_INVALID_ARG_TYPE. Functions are objects and the spec does not exclude them; the iteration protocol checks already accept them. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/streams/iter/utils.js | 5 ++++- test/parallel/test-stream-iter-from-async.js | 23 ++++++++++++++++++++ test/parallel/test-stream-iter-from-sync.js | 12 ++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/internal/streams/iter/utils.js b/lib/internal/streams/iter/utils.js index 34698dbdf0da..f84d663e9177 100644 --- a/lib/internal/streams/iter/utils.js +++ b/lib/internal/streams/iter/utils.js @@ -376,7 +376,10 @@ function toWriterUint8Array(chunk) { * @returns {boolean} */ function getProtocolMethod(value, symbol) { - if (value === null || typeof value !== 'object' || !(symbol in value)) { + // Functions are objects too, and may implement the protocols. + if (value === null || + (typeof value !== 'object' && typeof value !== 'function') || + !(symbol in value)) { return undefined; } const method = value[symbol]; diff --git a/test/parallel/test-stream-iter-from-async.js b/test/parallel/test-stream-iter-from-async.js index d04d7df9aad4..1b58fd6603ca 100644 --- a/test/parallel/test-stream-iter-from-async.js +++ b/test/parallel/test-stream-iter-from-async.js @@ -507,12 +507,35 @@ function testFromUndefinedThrows() { assert.throws(() => from(undefined), { code: 'ERR_INVALID_ARG_TYPE' }); } +async function testFromFunctionWithProtocols() { + // Functions are objects and may implement the protocols. + function asyncSource() {} + asyncSource[Symbol.for('Stream.toAsyncStreamable')] = + async () => 'async-function'; + assert.strictEqual(await text(from(asyncSource)), 'async-function'); + + function syncSource() {} + syncSource[Symbol.for('Stream.toStreamable')] = () => 'sync-function'; + assert.strictEqual(await text(from(syncSource)), 'sync-function'); + + async function* nested() { + yield asyncSource; + yield syncSource; + } + assert.strictEqual(await text(from(nested())), + 'async-functionsync-function'); + + // A function without a protocol is still rejected. + assert.throws(() => from(() => {}), { code: 'ERR_INVALID_ARG_TYPE' }); +} + Promise.all([ testFromString(), testFromAsyncGenerator(), testFromAsyncIteratorResultShapes(), testFromSourceErrorDoesNotWaitForReturn(), testFromBoundsNestedAsyncIterable(), + testFromFunctionWithProtocols(), testFromDoesNotHoldBackNestedAsyncIterable(), testFromBoundsPreBatchedAsyncValues(), testFromSyncIterableAsAsync(), diff --git a/test/parallel/test-stream-iter-from-sync.js b/test/parallel/test-stream-iter-from-sync.js index 50d85dc94a98..af7acfd4a546 100644 --- a/test/parallel/test-stream-iter-from-sync.js +++ b/test/parallel/test-stream-iter-from-sync.js @@ -239,6 +239,17 @@ function testFromSyncUndefinedThrows() { assert.throws(() => fromSync(undefined), { code: 'ERR_INVALID_ARG_TYPE' }); } +function testFromSyncFunctionWithToStreamable() { + // Functions are objects and may implement the protocol. + function source() {} + source[Symbol.for('Stream.toStreamable')] = () => 'from-function'; + assert.strictEqual(textSync(fromSync(source)), 'from-function'); + // ...also when nested inside another source. + assert.strictEqual(textSync(fromSync([source, '!'])), 'from-function!'); + // A function without a protocol is still rejected. + assert.throws(() => fromSync(() => {}), { code: 'ERR_INVALID_ARG_TYPE' }); +} + Promise.all([ testFromSyncString(), testFromSyncUint8Array(), @@ -260,4 +271,5 @@ Promise.all([ testFromSyncPrefersIteratorForThenableIterable(), testFromSyncRejectsPromise(), testFromSyncDataView(), + testFromSyncFunctionWithToStreamable(), ]).then(common.mustCall()); From dcac51a7db5b97e6ee44182a902f3742519e71af Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 14:57:51 +0000 Subject: [PATCH 74/98] stream: align pull() abort handling with the spec pull() threw synchronously when given an already-aborted signal. The spec (Iterable Streams, Stream.pull() step 5) requires it to return an iterable that throws the abort reason when read. This also matches how broadcast.push() and share.pull() already handle a pre-aborted signal. Once the signal aborted, the pull that observed the abort rejected, but later pulls resolved { done: true } because the pipeline is an async generator, which completes after throwing. The spec (step 7) requires future pulls to reject with the abort reason as well, so that a stream that was cancelled is never reported as having ended cleanly. Return an iterator that rejects every read with the abort reason once the signal has aborted the pipeline, without starting the pipeline if the signal was already aborted. Pipelines without a signal are not affected. The two tests that asserted the synchronous throw now check the rejection instead, and the documentation is updated. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/stream_iter.md | 8 +- lib/internal/streams/iter/pull.js | 97 +++++++++++++++----- test/parallel/test-stream-iter-pull-async.js | 74 +++++++++++++-- 3 files changed, 146 insertions(+), 33 deletions(-) diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index deb6a11bb85b..db23c97e8d20 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -713,8 +713,12 @@ added: v25.9.0 Create a lazy async pipeline. Source conversion and streamable protocol dispatch occur when `pull()` is called, but data is not read from `source` -until the returned iterable is consumed. A signal that is already aborted is -thrown synchronously after source conversion. Transforms are applied in order. +until the returned iterable is consumed. Transforms are applied in order. + +When `signal` aborts, the pending read (or the next one) rejects with +`signal.reason`, and so does every later read. If `signal` is already aborted, +`pull()` still returns an iterable; reading from it rejects with +`signal.reason` without reading from `source`. ```mjs import { from, pull, text } from 'node:stream/iter'; diff --git a/lib/internal/streams/iter/pull.js b/lib/internal/streams/iter/pull.js index 70354edb10b3..792e8d33e9c2 100644 --- a/lib/internal/streams/iter/pull.js +++ b/lib/internal/streams/iter/pull.js @@ -12,6 +12,7 @@ const { ArrayPrototypeSlice, FunctionPrototypeCall, PromisePrototypeThen, + PromiseReject, PromiseResolve, SymbolAsyncIterator, SymbolIterator, @@ -850,37 +851,83 @@ function pull(source, ...args) { }); const { signal } = options; const normalized = from(source); - signal?.throwIfAborted(); return { __proto__: null, [SymbolAsyncIterator]() { - const controller = new AbortController(); - const iteratorSignal = signal === undefined ? - controller.signal : AbortSignal.any([signal, controller.signal]); - - async function* pipeline() { - yield* createAsyncPipeline(normalized, transforms, iteratorSignal); + if (signal === undefined) { + const controller = new AbortController(); + async function* pipeline() { + yield* createAsyncPipeline(normalized, transforms, controller.signal); + } + const iterator = pipeline(); + return { + __proto__: null, + next(value) { + return iterator.next(value); + }, + return(value) { + controller.abort(lazyDOMException('Aborted', 'AbortError')); + return iterator.return(value); + }, + throw(error) { + abortSignal(controller.signal, error); + return iterator.throw(error); + }, + [SymbolAsyncIterator]() { + return this; + }, + }; } - const iterator = pipeline(); + return createAbortablePullIterator(normalized, transforms, signal); + }, + }; +} - return { - __proto__: null, - next(value) { - return iterator.next(value); - }, - return(value) { - controller.abort(lazyDOMException('Aborted', 'AbortError')); - return iterator.return(value); - }, - throw(error) { - abortSignal(controller.signal, error); - return iterator.throw(error); - }, - [SymbolAsyncIterator]() { - return this; - }, - }; +// Once `signal` aborts the pipeline, the pull that observed it rejects and +// so does every later pull, with the abort reason. That includes the case of +// an already-aborted signal, where the pipeline is never started. A plain +// async generator would instead complete after throwing, so later pulls +// would report a clean end of the stream. +function createAbortablePullIterator(source, transforms, signal) { + let aborted = signal.aborted; + let controller; + let iterator; + if (!aborted) { + controller = new AbortController(); + const iteratorSignal = AbortSignal.any([signal, controller.signal]); + async function* pipeline() { + yield* createAsyncPipeline(source, transforms, iteratorSignal); + } + iterator = pipeline(); + } + + function onRejected(error) { + if (signal.aborted) aborted = true; + throw error; + } + + return { + __proto__: null, + next(value) { + if (aborted) return PromiseReject(signal.reason); + return PromisePrototypeThen(iterator.next(value), undefined, onRejected); + }, + return(value) { + if (aborted) { + return PromiseResolve({ __proto__: null, done: true, value }); + } + controller.abort(lazyDOMException('Aborted', 'AbortError')); + return iterator.return(value); + }, + throw(error) { + if (aborted) return PromiseReject(error); + abortSignal(controller.signal, error); + return PromisePrototypeThen(iterator.throw(error), undefined, + onRejected); + }, + [SymbolAsyncIterator]() { + return this; }, }; } diff --git a/test/parallel/test-stream-iter-pull-async.js b/test/parallel/test-stream-iter-pull-async.js index 1a5d1eb28737..632306fa5895 100644 --- a/test/parallel/test-stream-iter-pull-async.js +++ b/test/parallel/test-stream-iter-pull-async.js @@ -66,14 +66,72 @@ async function testPullStatefulTransformReceiver() { } async function testPullWithAbortSignal() { + let started = false; async function* gen() { + started = true; yield [new Uint8Array([1])]; } - assert.throws( - () => pull(gen(), { signal: AbortSignal.abort() }), - { name: 'AbortError' }, - ); + // An already-aborted signal does not make pull() throw; the returned + // iterable rejects instead, without reading from the source. + const signal = AbortSignal.abort(); + const iterator = pull(gen(), { signal })[Symbol.asyncIterator](); + await assert.rejects(iterator.next(), (error) => error === signal.reason); + await assert.rejects(iterator.next(), (error) => error === signal.reason); + assert.strictEqual(started, false); + assert.deepStrictEqual(await iterator.return(), + { __proto__: null, done: true, value: undefined }); + + await assert.rejects(text(pull(gen(), (chunks) => chunks, { signal })), + (error) => error === signal.reason); + assert.strictEqual(started, false); +} + +async function testPullKeepsRejectingAfterAbort() { + for (const transforms of [[], [(chunks) => chunks]]) { + // Abort while a read is pending. + { + const ac = new AbortController(); + const reason = new Error('stop'); + async function* gen() { + yield [new Uint8Array([1])]; + await new Promise(() => {}); + } + const iterator = + pull(gen(), ...transforms, { signal: ac.signal })[Symbol.asyncIterator](); + assert.strictEqual((await iterator.next()).done, false); + const pending = iterator.next(); + ac.abort(reason); + await assert.rejects(pending, (error) => error === reason); + await assert.rejects(iterator.next(), (error) => error === reason); + await assert.rejects(iterator.next(), (error) => error === reason); + } + // Abort between reads. + { + const ac = new AbortController(); + const reason = new Error('stop'); + async function* gen() { + yield [new Uint8Array([1])]; + yield [new Uint8Array([2])]; + } + const iterator = + pull(gen(), ...transforms, { signal: ac.signal })[Symbol.asyncIterator](); + assert.strictEqual((await iterator.next()).done, false); + ac.abort(reason); + await assert.rejects(iterator.next(), (error) => error === reason); + await assert.rejects(iterator.next(), (error) => error === reason); + } + // Aborting after the pipeline completed does not change the result. + { + const ac = new AbortController(); + const iterator = + pull(from('x'), ...transforms, { signal: ac.signal })[Symbol.asyncIterator](); + assert.strictEqual((await iterator.next()).done, false); + assert.strictEqual((await iterator.next()).done, true); + ac.abort(); + assert.strictEqual((await iterator.next()).done, true); + } + } } async function testPullNormalizesSourceAtCallTime() { @@ -99,7 +157,7 @@ async function testPullNormalizesSourceAtCallTime() { assert.strictEqual(iteratorCalls, 1); } -function testPullPreAbortOrdering() { +async function testPullPreAbortOrdering() { const reason = new Error('already aborted'); let protocolCalls = 0; const source = { @@ -110,8 +168,11 @@ function testPullPreAbortOrdering() { }; const signal = AbortSignal.abort(reason); - assert.throws(() => pull(source, { signal }), (error) => error === reason); + // Source conversion still happens when pull() is called; the abort is + // reported when the result is read. + const result = pull(source, { signal }); assert.strictEqual(protocolCalls, 1); + await assert.rejects(text(result), (error) => error === reason); assert.throws( () => pull(null, { signal }), { code: 'ERR_INVALID_ARG_TYPE' }, @@ -521,6 +582,7 @@ async function testTransformOptionsNotShared() { testPullStatefulTransform(), testPullStatefulTransformReceiver(), testPullWithAbortSignal(), + testPullKeepsRejectingAfterAbort(), testPullNormalizesSourceAtCallTime(), testPullPreAbortOrdering(), testPullChainedTransforms(), From 7ba60dcdbe561cc8c30228c27f2fad1b9e316d75 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 15:00:10 +0000 Subject: [PATCH 75/98] stream: keep the original error when writer.fail() throws When pipeTo() or pipeToSync() failed, they called writer.fail(error) and then rethrew the error. If fail() itself threw, its exception replaced the error that made the pipe fail, which was then lost. Call fail() on a best-effort basis and always surface the original error. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/streams/iter/pull.js | 15 ++++++- test/parallel/test-stream-iter-pipeto-edge.js | 39 ++++++++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/lib/internal/streams/iter/pull.js b/lib/internal/streams/iter/pull.js index 792e8d33e9c2..878fa4202390 100644 --- a/lib/internal/streams/iter/pull.js +++ b/lib/internal/streams/iter/pull.js @@ -1083,7 +1083,7 @@ function pipeToSync(source, ...args) { } } catch (error) { if (!options.preventFail) { - writer.fail?.(error); + failWriterQuietly(writer, error); } throw error; } @@ -1091,6 +1091,17 @@ function pipeToSync(source, ...args) { return totalBytes; } +// Call writer.fail(error) on a best-effort basis. The error that made the +// pipe fail is what the caller must see; an exception from fail() itself +// must not replace it. +function failWriterQuietly(writer, error) { + try { + writer.fail?.(error); + } catch { + // Ignored, see above. + } +} + /** * Write an async source through transforms to a writer. * @param {AsyncIterable|Iterable} source @@ -1108,7 +1119,7 @@ async function pipeTo(source, ...args) { function failWriter(error) { if (!options.preventFail) { - writer.fail?.(error); + failWriterQuietly(writer, error); } } diff --git a/test/parallel/test-stream-iter-pipeto-edge.js b/test/parallel/test-stream-iter-pipeto-edge.js index 13223a226a1f..d63171203d6d 100644 --- a/test/parallel/test-stream-iter-pipeto-edge.js +++ b/test/parallel/test-stream-iter-pipeto-edge.js @@ -5,7 +5,7 @@ const common = require('../common'); const assert = require('assert'); -const { pipeToSync, fromSync } = require('stream/iter'); +const { pipeTo, pipeToSync, fromSync } = require('stream/iter'); // pipeToSync cannot complete when endSync() requires async fallback. async function testPipeToSyncEndSyncFailure() { @@ -68,9 +68,46 @@ async function testPipeToSyncPreventClose() { assert.strictEqual(endCalled, false); } +// An exception thrown by writer.fail() must not replace the error that made +// the pipe fail. +async function testFailThrowingDoesNotMaskError() { + const cause = new Error('write failed'); + const syncWriter = { + writeSync() { throw cause; }, + endSync: common.mustNotCall(), + fail: common.mustCall((error) => { + assert.strictEqual(error, cause); + throw new Error('fail() threw'); + }), + }; + assert.throws(() => pipeToSync(fromSync('data'), syncWriter), + (error) => error === cause); + + const asyncWriter = { + async write() { throw cause; }, + end: common.mustNotCall(), + fail: common.mustCall((error) => { + assert.strictEqual(error, cause); + throw new Error('fail() threw'); + }), + }; + await assert.rejects(pipeTo(fromSync('data'), asyncWriter), + (error) => error === cause); + + // Same for the already-aborted signal path of pipeTo(). + const signal = AbortSignal.abort(); + await assert.rejects( + pipeTo(fromSync('data'), { + write: common.mustNotCall(), + fail() { throw new Error('fail() threw'); }, + }, { signal }), + (error) => error === signal.reason); +} + Promise.all([ testPipeToSyncEndSyncFailure(), testPipeToSyncNoEndSync(), testPipeToSyncPreventFail(), testPipeToSyncPreventClose(), + testFailThrowingDoesNotMaskError(), ]).then(common.mustCall()); From f302fbd04a2c23c2b4a439263b5c75ca1e593979 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 15:05:15 +0000 Subject: [PATCH 76/98] stream: reduce per-chunk overhead in stream/iter consumers bytes(), text(), arrayBuffer(), array() and their sync variants kept a snapshot object for every collected chunk, plus a batch entry and its views array, so they can detect chunks that were resized or detached before the result is assembled. For streams of many small chunks this dominated memory use: collecting 1,000,000 one-byte chunks peaked at about 490 MB of heap for 1 MB of data. A non-empty view of a fixed-length, non-shared ArrayBuffer can only change by its buffer being detached, which makes its byteLength 0, so recording its byteLength is enough. Keep a full snapshot only for empty views and views of resizable or shared buffers. The same input now peaks at about 60 MB and is collected about five times faster. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/streams/iter/consumers.js | 55 +++++++------------ lib/internal/streams/iter/utils.js | 48 ++++++++++++++++ .../test-stream-iter-resizable-buffers.js | 30 ++++++++++ 3 files changed, 98 insertions(+), 35 deletions(-) diff --git a/lib/internal/streams/iter/consumers.js b/lib/internal/streams/iter/consumers.js index d2ecf5f3e21b..a18794aa7323 100644 --- a/lib/internal/streams/iter/consumers.js +++ b/lib/internal/streams/iter/consumers.js @@ -54,9 +54,9 @@ const { const { concatBytes, - createBatchEntry, getProtocolMethod, - validateBatchEntry, + recordChunk, + validateRecordedChunks, yieldAbortable, } = require('internal/streams/iter/utils'); @@ -95,17 +95,6 @@ function isMergeOptions(value) { // Shared chunk collection helpers // ============================================================================= -function flattenBatchEntries(entries) { - const chunks = []; - for (let i = 0; i < entries.length; i++) { - const batch = validateBatchEntry(entries[i]); - for (let j = 0; j < batch.length; j++) { - ArrayPrototypePush(chunks, batch[j]); - } - } - return chunks; -} - /** * Collect chunks from a sync source into an array. * @param {Iterable} source @@ -115,23 +104,20 @@ function flattenBatchEntries(entries) { function collectSync(source, limit) { // Normalize source via fromSync() - accepts strings, ArrayBuffers, protocols, etc. const normalized = fromSync(source); - const entries = []; + const chunks = []; + const checks = []; let totalBytes = 0; for (const batch of normalized) { - const entry = createBatchEntry(batch); - if (limit !== undefined) { - for (let i = 0; i < entry.views.length; i++) { - totalBytes += entry.views[i].byteLength; - if (totalBytes > limit) { - throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); - } + for (let i = 0; i < batch.length; i++) { + totalBytes += recordChunk(chunks, checks, batch[i]); + if (limit !== undefined && totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); } } - ArrayPrototypePush(entries, entry); } - return flattenBatchEntries(entries); + return validateRecordedChunks(chunks, checks); } /** @@ -146,14 +132,17 @@ async function collectAsync(source, signal, limit) { // Normalize source via from() - accepts strings, ArrayBuffers, protocols, etc. const normalized = from(source); - const entries = []; + const chunks = []; + const checks = []; // Fast path: no signal and no limit if (!signal && limit === undefined) { for await (const batch of normalized) { - ArrayPrototypePush(entries, createBatchEntry(batch)); + for (let i = 0; i < batch.length; i++) { + recordChunk(chunks, checks, batch[i]); + } } - return flattenBatchEntries(entries); + return validateRecordedChunks(chunks, checks); } // Slow path: with signal or limit checks @@ -162,19 +151,15 @@ async function collectAsync(source, signal, limit) { for await (const batch of iterable) { signal?.throwIfAborted(); - const entry = createBatchEntry(batch); - if (limit !== undefined) { - for (let i = 0; i < entry.views.length; i++) { - totalBytes += entry.views[i].byteLength; - if (totalBytes > limit) { - throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); - } + for (let i = 0; i < batch.length; i++) { + totalBytes += recordChunk(chunks, checks, batch[i]); + if (limit !== undefined && totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); } } - ArrayPrototypePush(entries, entry); } - return flattenBatchEntries(entries); + return validateRecordedChunks(chunks, checks); } /** diff --git a/lib/internal/streams/iter/utils.js b/lib/internal/streams/iter/utils.js index f84d663e9177..99e5c5cd0935 100644 --- a/lib/internal/streams/iter/utils.js +++ b/lib/internal/streams/iter/utils.js @@ -4,6 +4,7 @@ const { Array, ArrayBufferPrototypeGetByteLength, ArrayBufferPrototypeGetDetached, + ArrayBufferPrototypeGetResizable, ArrayPrototypePush, ArrayPrototypeSlice, PromiseResolve, @@ -235,6 +236,51 @@ function validateByteView(snapshot) { return value; } +/** + * Append a chunk to `chunks` for later concatenation, and the information + * needed to detect that it was resized or detached in the meantime to + * `checks`. This provides the same guarantee as snapshotByteView() and + * validateByteView() without allocating a snapshot for every chunk in the + * common case: a non-empty view of a fixed-length, non-shared ArrayBuffer can + * only change by the buffer being detached, which makes its byteLength 0, so + * its byteLength is all that needs to be recorded. + * @param {Uint8Array[]} chunks + * @param {Array} checks + * @param {Uint8Array} value + * @returns {number} The byteLength of `value`. + */ +function recordChunk(chunks, checks, value) { + const buffer = TypedArrayPrototypeGetBuffer(value); + const byteLength = TypedArrayPrototypeGetByteLength(value); + ArrayPrototypePush(chunks, value); + if (byteLength === 0 || isSharedArrayBuffer(buffer) || + ArrayBufferPrototypeGetResizable(buffer)) { + ArrayPrototypePush(checks, snapshotByteView(value)); + } else { + ArrayPrototypePush(checks, byteLength); + } + return byteLength; +} + +/** + * Validate chunks recorded with recordChunk(). + * @param {Uint8Array[]} chunks + * @param {Array} checks + * @returns {Uint8Array[]} `chunks` + */ +function validateRecordedChunks(chunks, checks) { + for (let i = 0; i < chunks.length; i++) { + const check = checks[i]; + if (typeof check !== 'number') { + validateByteView(check); + } else if (TypedArrayPrototypeGetByteLength(chunks[i]) !== check) { + throw new ERR_INVALID_STATE.TypeError( + 'Byte view was resized or detached after being accepted'); + } + } + return chunks; +} + function createBatchEntry(chunks) { const views = new Array(chunks.length); let byteLength = 0; @@ -489,6 +535,7 @@ module.exports = { concatBytes, convertChunks, createBatchEntry, + recordChunk, splitBatchEntry, getProtocolMethod, getWriterSignal, @@ -503,6 +550,7 @@ module.exports = { toWriterUint8Array, validateBackpressure, validateBatchEntry, + validateRecordedChunks, validateByteView, yieldAbortable, }; diff --git a/test/parallel/test-stream-iter-resizable-buffers.js b/test/parallel/test-stream-iter-resizable-buffers.js index b66973393cac..91f7429ec12f 100644 --- a/test/parallel/test-stream-iter-resizable-buffers.js +++ b/test/parallel/test-stream-iter-resizable-buffers.js @@ -167,6 +167,35 @@ async function testPipeRejectsWriterResize() { ); } +async function testConsumersRejectDetachedViews() { + // Views of fixed-length buffers are tracked without a full snapshot; they + // must still be rejected when detached after being accepted. + const asyncBuffer = new ArrayBuffer(2); + async function* asyncSource() { + yield [new Uint8Array(asyncBuffer), Uint8Array.of(1)]; + asyncBuffer.transfer(); + } + await assert.rejects(array(asyncSource()), kResizeError); + const limitedBuffer = new ArrayBuffer(2); + async function* limitedSource() { + yield [new Uint8Array(limitedBuffer)]; + limitedBuffer.transfer(); + } + await assert.rejects(array(limitedSource(), { limit: 10 }), kResizeError); + + const syncBuffer = new ArrayBuffer(2); + function* syncSource() { + yield [new Uint8Array(syncBuffer, 1)]; + syncBuffer.transfer(); + } + assert.throws(() => arraySync(syncSource()), kResizeError); + + // Unchanged views are returned as-is. + const chunk = new Uint8Array(4); + const [result] = arraySync([[chunk]]); + assert.strictEqual(result, chunk); +} + Promise.all([ testBufferedViewMutationRejected(), testDropOldestUsesAcceptedByteLength(), @@ -174,5 +203,6 @@ Promise.all([ testBroadcastRejectsResizedBufferedView(), testShareRejectsResizedBufferedView(), testConsumersRejectResizedViews(), + testConsumersRejectDetachedViews(), testPipeRejectsWriterResize(), ]).then(common.mustCall()); From 4b83c6c38e501703f289491909c24e944e2c69b0 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 15:08:04 +0000 Subject: [PATCH 77/98] stream: accept explicit stream/iter budgets below 16384 push(), duplex(), broadcast(), share() and shareSync() rejected an explicit budget below 16384 bytes with ERR_OUT_OF_RANGE. The spec (push() step 3, broadcast() step 1 and share() step 2) only requires the implementation-defined default to be at least 16384 bytes; an explicit budget is used as given. Small budgets are also useful in tests and in memory-constrained code. Accept any explicit budget of at least 1 byte, and validate it in one place. Defaults are unchanged. The validation tests are updated to the new lower bound; note that WebIDL conversion truncates fractions, so 1.5 is now a valid budget of 1 and 0.5 is used to exercise the rejection instead. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/stream_iter.md | 10 ++--- lib/internal/streams/iter/broadcast.js | 6 +-- lib/internal/streams/iter/push.js | 6 +-- lib/internal/streams/iter/share.js | 8 ++-- lib/internal/streams/iter/utils.js | 11 +++++ .../test-stream-iter-push-backpressure.js | 17 ++++++++ test/parallel/test-stream-iter-validation.js | 42 ++++++++++--------- 7 files changed, 62 insertions(+), 38 deletions(-) diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index db23c97e8d20..9f28707f7105 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -801,7 +801,7 @@ added: v25.9.0 readable side. * `options` {Object} * `budget` {number} Maximum number of buffered bytes before - backpressure is applied. Must be >= 16384. + backpressure is applied. Must be a positive integer. **Default:** `16384`. * `backpressure` {string} Backpressure policy: `'strict'`, `'unbounded'`, `'drop-oldest'`, or `'drop-newest'`. **Default:** `'strict'`. @@ -865,7 +865,7 @@ added: v25.9.0 * `options` {Object} * `budget` {number} Buffer size in bytes for both directions. - Must be >= 16384. **Default:** `16384`. + Must be a positive integer. **Default:** `16384`. * `backpressure` {string} Policy for both directions. **Default:** `'strict'`. * `signal` {AbortSignal} Cancellation signal for both channels. @@ -1323,7 +1323,7 @@ added: v25.9.0 --> * `options` {Object} - * `budget` {number} Buffer size in bytes. Must be >= 16384. + * `budget` {number} Buffer size in bytes. Must be a positive integer. **Default:** `65536`. * `backpressure` {string} `'strict'`, `'unbounded'`, `'drop-oldest'`, or `'drop-newest'`. **Default:** `'strict'`. @@ -1433,7 +1433,7 @@ added: v25.9.0 * `source` {AsyncIterable} The source to share. * `options` {Object} - * `budget` {number} Buffer size in bytes. Must be >= 16384. + * `budget` {number} Buffer size in bytes. Must be a positive integer. **Default:** `65536`. * `backpressure` {string} `'strict'`, `'unbounded'`, `'drop-oldest'`, or `'drop-newest'`. **Default:** `'strict'`. @@ -1559,7 +1559,7 @@ added: v25.9.0 * `source` {Iterable} The sync source to share. * `options` {Object} - * `budget` {number} Must be >= 16384. + * `budget` {number} Must be a positive integer. **Default:** `65536`. * `backpressure` {string} `'strict'` or `'drop-oldest'`. **Default:** `'strict'`. diff --git a/lib/internal/streams/iter/broadcast.js b/lib/internal/streams/iter/broadcast.js index e32f8aa544d2..cdf25021bbc5 100644 --- a/lib/internal/streams/iter/broadcast.js +++ b/lib/internal/streams/iter/broadcast.js @@ -40,9 +40,6 @@ const { ERR_INVALID_STATE, }, } = require('internal/errors'); -const { - validateInteger, -} = require('internal/validators'); const { broadcastProtocol, @@ -72,6 +69,7 @@ const { toWriterUint8Array, validateBatchEntry, yieldAbortable, + validateBudget, } = require('internal/streams/iter/utils'); const { converters, @@ -895,7 +893,7 @@ function broadcast(options = { __proto__: null }) { backpressure = 'strict', signal, } = options; - validateInteger(budget, 'options.budget', 16384); + validateBudget(budget); const opts = { __proto__: null, diff --git a/lib/internal/streams/iter/push.js b/lib/internal/streams/iter/push.js index 31b437c85b75..764a3afac92a 100644 --- a/lib/internal/streams/iter/push.js +++ b/lib/internal/streams/iter/push.js @@ -22,9 +22,6 @@ const { ERR_INVALID_STATE, }, } = require('internal/errors'); -const { - validateInteger, -} = require('internal/validators'); const { drainableProtocol, @@ -40,6 +37,7 @@ const { getWriterSignal, parsePullArgs, validateBatchEntry, + validateBudget, } = require('internal/streams/iter/utils'); const { converters, @@ -132,7 +130,7 @@ class PushQueue { backpressure = 'strict', signal, } = options; - validateInteger(budget, 'options.budget', 16384); + validateBudget(budget); this.#budget = budget; this.#backpressure = backpressure; this.#signal = signal; diff --git a/lib/internal/streams/iter/share.js b/lib/internal/streams/iter/share.js index 17dabf5da039..6e9881b682a4 100644 --- a/lib/internal/streams/iter/share.js +++ b/lib/internal/streams/iter/share.js @@ -46,6 +46,7 @@ const { parsePullArgs, splitBatchEntry, validateBatchEntry, + validateBudget, } = require('internal/streams/iter/utils'); const { converters, @@ -64,9 +65,6 @@ const { ERR_OUT_OF_RANGE, }, } = require('internal/errors'); -const { - validateInteger, -} = require('internal/validators'); const { markPromiseAsHandled } = internalBinding('util'); @@ -865,7 +863,7 @@ function share(source, options = { __proto__: null }) { backpressure = 'strict', signal, } = options; - validateInteger(budget, 'options.budget', 16384); + validateBudget(budget); const opts = { __proto__: null, @@ -894,7 +892,7 @@ function shareSync(source, options = { __proto__: null }) { budget = kMultiConsumerDefaultBudget, backpressure = 'strict', } = options; - validateInteger(budget, 'options.budget', 16384); + validateBudget(budget); // A synchronous consumer can neither wait for the slowest consumer to // release budget ('unbounded') nor keep pulling and discarding until it // does ('drop-newest'): the slowest consumer cannot advance while the diff --git a/lib/internal/streams/iter/utils.js b/lib/internal/streams/iter/utils.js index 99e5c5cd0935..2f72a78c0d21 100644 --- a/lib/internal/streams/iter/utils.js +++ b/lib/internal/streams/iter/utils.js @@ -35,6 +35,7 @@ const { const { isSharedArrayBuffer, isUint8Array } = require('internal/util/types'); const { + validateInteger, validateOneOf, } = require('internal/validators'); const { @@ -236,6 +237,15 @@ function validateByteView(snapshot) { return value; } +/** + * Validate an explicit `budget` option. The spec only requires the default + * budget to be at least 16384 bytes; any positive explicit budget is valid. + * @param {unknown} budget + */ +function validateBudget(budget) { + validateInteger(budget, 'options.budget', 1); +} + /** * Append a chunk to `chunks` for later concatenation, and the information * needed to detect that it was resized or detached in the meantime to @@ -550,6 +560,7 @@ module.exports = { toWriterUint8Array, validateBackpressure, validateBatchEntry, + validateBudget, validateRecordedChunks, validateByteView, yieldAbortable, diff --git a/test/parallel/test-stream-iter-push-backpressure.js b/test/parallel/test-stream-iter-push-backpressure.js index 9fb3325f3423..58ba2d331dcb 100644 --- a/test/parallel/test-stream-iter-push-backpressure.js +++ b/test/parallel/test-stream-iter-push-backpressure.js @@ -150,6 +150,22 @@ async function testStrictPendingQueueOverflow() { await iter.return(); } +async function testSmallBudget() { + // An explicit budget below the 16384-byte default is honored. + const { writer, readable } = push({ budget: 4 }); + assert.strictEqual(writer.writeSync('abc'), true); + assert.strictEqual(writer.canWrite, true); + // The buffer may overshoot the budget by one write. + assert.strictEqual(writer.writeSync('defgh'), true); + assert.strictEqual(writer.canWrite, false); + assert.strictEqual(writer.writeSync('i'), false); + const write = writer.write('ij'); + const result = text(readable); + await write; + writer.endSync(); + assert.strictEqual(await result, 'abcdefghij'); +} + Promise.all([ testStrictBackpressure(), testDropOldest(), @@ -157,4 +173,5 @@ Promise.all([ testBlockBackpressure(), testBlockWriteSyncDoesNotEnqueue(), testStrictPendingQueueOverflow(), + testSmallBudget(), ]).then(common.mustCall()); diff --git a/test/parallel/test-stream-iter-validation.js b/test/parallel/test-stream-iter-validation.js index b130a92ae3de..16ff12c060e7 100644 --- a/test/parallel/test-stream-iter-validation.js +++ b/test/parallel/test-stream-iter-validation.js @@ -21,15 +21,18 @@ const { // push() validation // ============================================================================= -// Budget must be integer >= 16384 +// Budget must be an integer >= 1 assert.throws(() => push({ budget: 'bad' }), { code: 'ERR_OUT_OF_RANGE' }); -assert.throws(() => push({ budget: 1.5 }), { code: 'ERR_OUT_OF_RANGE' }); -// Values < 16384 are rejected +// WebIDL conversion truncates fractions: 0.5 becomes 0, 1.5 becomes 1. +assert.throws(() => push({ budget: 0.5 }), { code: 'ERR_OUT_OF_RANGE' }); +assert.strictEqual(push({ budget: 1.5 }).writer.canWrite, true); +// Values < 1 are rejected assert.throws(() => push({ budget: 0 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => push({ budget: -1 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => push({ budget: -100 }), { code: 'ERR_OUT_OF_RANGE' }); -assert.throws(() => push({ budget: 16383 }), { code: 'ERR_OUT_OF_RANGE' }); -// 16384 is the minimum accepted value +// Only the default must be at least 16384; smaller explicit budgets are valid +assert.strictEqual(push({ budget: 1 }).writer.canWrite, true); +assert.strictEqual(push({ budget: 16383 }).writer.canWrite, true); assert.strictEqual(push({ budget: 16384 }).writer.canWrite, true); // MAX_SAFE_INTEGER is accepted assert.strictEqual(push({ budget: Number.MAX_SAFE_INTEGER }).writer.canWrite, @@ -87,11 +90,11 @@ assert.throws(() => duplex({ b: 'bad' }), { code: 'ERR_INVALID_ARG_TYPE' }); // Budget validation (cascades through to push()) assert.throws(() => duplex({ budget: 'bad' }), { code: 'ERR_OUT_OF_RANGE' }); -assert.throws(() => duplex({ budget: 1.5 }), { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => duplex({ budget: 0.5 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => duplex({ budget: Number.MAX_SAFE_INTEGER + 1 }), { code: 'ERR_OUT_OF_RANGE' }); -// Values < 16384 are rejected (both directions) +// Values < 1 are rejected (both directions) assert.throws(() => duplex({ budget: 0 }), { code: 'ERR_OUT_OF_RANGE' }); // MAX_SAFE_INTEGER is accepted { @@ -129,17 +132,16 @@ assert.throws(() => pullSync(fromSync('a'), 42), { code: 'ERR_INVALID_ARG_TYPE' // ============================================================================= assert.throws(() => broadcast({ budget: 'bad' }), { code: 'ERR_OUT_OF_RANGE' }); -assert.throws(() => broadcast({ budget: 1.5 }), { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => broadcast({ budget: 0.5 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => broadcast({ budget: Number.MAX_SAFE_INTEGER + 1 }), { code: 'ERR_OUT_OF_RANGE' }); -// Values < 16384 are rejected +// Values < 1 are rejected assert.throws(() => broadcast({ budget: 0 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => broadcast({ budget: -1 }), { code: 'ERR_OUT_OF_RANGE' }); -assert.throws(() => broadcast({ budget: 16383 }), { code: 'ERR_OUT_OF_RANGE' }); -// 16384 is the minimum accepted value +// Small explicit budgets are accepted { - const bc = broadcast({ budget: 16384 }); + const bc = broadcast({ budget: 1 }); bc.broadcast.push(); assert.strictEqual(bc.writer.canWrite, true); bc.writer.endSync(); @@ -235,7 +237,7 @@ assert.throws( assert.throws(() => share(42), { code: 'ERR_INVALID_ARG_TYPE' }); assert.throws(() => share(from('a'), { budget: 'bad' }), { code: 'ERR_OUT_OF_RANGE' }); -assert.throws(() => share(from('a'), { budget: 1.5 }), { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => share(from('a'), { budget: 0.5 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => share(from('a'), { budget: Number.MAX_SAFE_INTEGER + 1 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => share(from('a'), { signal: {} }), { code: 'ERR_INVALID_ARG_TYPE' }); @@ -250,28 +252,28 @@ assert.throws(() => share(from('a'), { backpressure: 'bad' }), { code: 'ERR_INVA assert.strictEqual(shared.consumerCount, 0); } -// share() values < 16384 are rejected +// share() values < 1 are rejected assert.throws(() => share(from('a'), { budget: 0 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => share(from('a'), { budget: -1 }), { code: 'ERR_OUT_OF_RANGE' }); -// 16384 is the minimum, MAX_SAFE_INTEGER is accepted -share(from('a'), { budget: 16384 }).cancel(); +// 1 is the minimum, MAX_SAFE_INTEGER is accepted +share(from('a'), { budget: 1 }).cancel(); share(from('a'), { budget: Number.MAX_SAFE_INTEGER }).cancel(); assert.throws(() => shareSync(42), { code: 'ERR_INVALID_ARG_TYPE' }); assert.throws(() => shareSync(fromSync('a'), { budget: 'bad' }), { code: 'ERR_OUT_OF_RANGE' }); -assert.throws(() => shareSync(fromSync('a'), { budget: 1.5 }), +assert.throws(() => shareSync(fromSync('a'), { budget: 0.5 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => shareSync(fromSync('a'), { budget: Number.MAX_SAFE_INTEGER + 1 }), { code: 'ERR_OUT_OF_RANGE' }); -// shareSync() values < 16384 are rejected +// shareSync() values < 1 are rejected assert.throws(() => shareSync(fromSync('a'), { budget: 0 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => shareSync(fromSync('a'), { budget: -1 }), { code: 'ERR_OUT_OF_RANGE' }); -// 16384 is the minimum, MAX_SAFE_INTEGER is accepted -shareSync(fromSync('a'), { budget: 16384 }).cancel(); +// 1 is the minimum, MAX_SAFE_INTEGER is accepted +shareSync(fromSync('a'), { budget: 1 }).cancel(); shareSync(fromSync('a'), { budget: Number.MAX_SAFE_INTEGER }).cancel(); // Share.from / SyncShare.fromSync reject non-iterable From af41622ad879cedcfa1cfc39103679c0575d1fa8 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 15:10:07 +0000 Subject: [PATCH 78/98] stream: reject closed fromWritable() writes with a TypeError The spec (Writer write(), step 2) requires writes to a closed writer to reject with a TypeError. The fromWritable() adapter rejected write() and writev() with ERR_STREAM_WRITE_AFTER_END, which is a plain Error, unlike the other stream/iter writers. Add a TypeError variant of ERR_STREAM_WRITE_AFTER_END and use it, so the error code is unchanged. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/errors.js | 2 +- lib/internal/streams/iter/classic.js | 4 ++-- test/parallel/test-stream-iter-from-writable-lifecycle.js | 4 ++-- test/parallel/test-stream-iter-writable-interop.js | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/internal/errors.js b/lib/internal/errors.js index e239fe21f509..2d8551302bd0 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1845,7 +1845,7 @@ E('ERR_STREAM_UNABLE_TO_PIPE', 'Cannot pipe to a closed or destroyed stream', Er E('ERR_STREAM_UNSHIFT_AFTER_END_EVENT', 'stream.unshift() after end event', Error); E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error); -E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error); +E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error, TypeError); E('ERR_SYNTHETIC', 'JavaScript Callstack', Error); E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError, HideStackFramesError); E('ERR_TEST_FAILURE', function(error, failureType) { diff --git a/lib/internal/streams/iter/classic.js b/lib/internal/streams/iter/classic.js index 59eb5be5819b..2c1fa6407149 100644 --- a/lib/internal/streams/iter/classic.js +++ b/lib/internal/streams/iter/classic.js @@ -912,7 +912,7 @@ function fromWritable(writable, options = kNullPrototype) { syncWritableError(); if (errored) return PromiseReject(error); if (!isWritable()) { - return PromiseReject(new ERR_STREAM_WRITE_AFTER_END()); + return PromiseReject(new ERR_STREAM_WRITE_AFTER_END.TypeError()); } if (signal?.aborted) return PromiseReject(signal.reason); @@ -944,7 +944,7 @@ function fromWritable(writable, options = kNullPrototype) { syncWritableError(); if (errored) return PromiseReject(error); if (!isWritable()) { - return PromiseReject(new ERR_STREAM_WRITE_AFTER_END()); + return PromiseReject(new ERR_STREAM_WRITE_AFTER_END.TypeError()); } if (signal?.aborted) return PromiseReject(signal.reason); if (chunks.length === 0) return PromiseResolve(); diff --git a/test/parallel/test-stream-iter-from-writable-lifecycle.js b/test/parallel/test-stream-iter-from-writable-lifecycle.js index cf6732c4184f..d5be23fdd03d 100644 --- a/test/parallel/test-stream-iter-from-writable-lifecycle.js +++ b/test/parallel/test-stream-iter-from-writable-lifecycle.js @@ -202,7 +202,7 @@ async function testAlreadyFinished() { const writer = fromWritable(writable); assert.strictEqual(writer.canWrite, null); await assert.rejects(writer.write('a'), - { code: 'ERR_STREAM_WRITE_AFTER_END' }); + { code: 'ERR_STREAM_WRITE_AFTER_END', name: 'TypeError' }); assert.strictEqual(await writer.end(), 0); assertNoTerminalListeners(writable); } @@ -215,7 +215,7 @@ async function testAlreadyDestroyed() { const writer = fromWritable(writable); assert.strictEqual(writer.canWrite, null); await assert.rejects(writer.write('a'), - { code: 'ERR_STREAM_WRITE_AFTER_END' }); + { code: 'ERR_STREAM_WRITE_AFTER_END', name: 'TypeError' }); assertNoTerminalListeners(writable); } diff --git a/test/parallel/test-stream-iter-writable-interop.js b/test/parallel/test-stream-iter-writable-interop.js index af2e05aae94c..f00aae924259 100644 --- a/test/parallel/test-stream-iter-writable-interop.js +++ b/test/parallel/test-stream-iter-writable-interop.js @@ -718,7 +718,7 @@ async function testWriteAfterEnd() { await assert.rejects( writer.write('should fail'), - { code: 'ERR_STREAM_WRITE_AFTER_END' }, + { code: 'ERR_STREAM_WRITE_AFTER_END', name: 'TypeError' }, ); } From 9648a6ba8e55788ee5ec4f61e10c9ce4b8fb7bd8 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 15:10:07 +0000 Subject: [PATCH 79/98] quic: reject closed stream writer writes with a TypeError The spec for stream/iter Writers (write(), step 2) requires writes to a closed writer to reject with a TypeError, as the push(), broadcast() and FileHandle writers do. The QUIC stream writer rejected write() and writev() with a plain ERR_INVALID_STATE Error. Use its TypeError variant; the error code is unchanged. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/quic/quic.js | 6 +++--- test/parallel/test-quic-stream-writer-api.mjs | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/internal/quic/quic.js b/lib/internal/quic/quic.js index 633e1f4e4dec..d163311dc182 100644 --- a/lib/internal/quic/quic.js +++ b/lib/internal/quic/quic.js @@ -2332,7 +2332,7 @@ class QuicStream { async function writeAsync(chunk, signal) { if (errored) throw error; if (closed || ending || stream.#inner.state.writeEnded) { - throw new ERR_INVALID_STATE('Writer is closed'); + throw new ERR_INVALID_STATE.TypeError('Writer is closed'); } signal?.throwIfAborted(); @@ -2358,7 +2358,7 @@ class QuicStream { await waitForDrain(signal); if (errored) throw error; if (closed || stream.#inner.state.writeEnded) { - throw new ERR_INVALID_STATE('Writer is closed'); + throw new ERR_INVALID_STATE.TypeError('Writer is closed'); } signal?.throwIfAborted(); if (writeConverted(chunks, token)) return; @@ -2402,7 +2402,7 @@ class QuicStream { async function writevAsync(chunks, signal) { if (errored) throw error; if (closed || ending || stream.#inner.state.writeEnded) { - throw new ERR_INVALID_STATE('Writer is closed'); + throw new ERR_INVALID_STATE.TypeError('Writer is closed'); } signal?.throwIfAborted(); diff --git a/test/parallel/test-quic-stream-writer-api.mjs b/test/parallel/test-quic-stream-writer-api.mjs index 352ee29ef594..26f240e4878d 100644 --- a/test/parallel/test-quic-stream-writer-api.mjs +++ b/test/parallel/test-quic-stream-writer-api.mjs @@ -104,6 +104,11 @@ await clientSession.opened; { code: 'ERR_INVALID_ARG_TYPE' }, ); assert.strictEqual(w.endSync(), 12); + // Writes after the writer is closed reject with a TypeError. + await assert.rejects(w.write('closed'), + { code: 'ERR_INVALID_STATE', name: 'TypeError' }); + await assert.rejects(w.writev(['closed']), + { code: 'ERR_INVALID_STATE', name: 'TypeError' }); for await (const _ of stream) { /* drain */ } // eslint-disable-line no-unused-vars await stream.closed; } From 16833534b6042062f922bf1e3c078c01abb8a792 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 15:11:20 +0000 Subject: [PATCH 80/98] stream: fix fromSync() async input error messages The ERR_INVALID_ARG_TYPE messages for async iterable and promise inputs read "must be an a synchronous input (not AsyncIterable)", because the error formatter adds "an" to expected-type strings that contain uppercase letters. Rephrase them in lowercase. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- lib/internal/streams/iter/from.js | 4 ++-- test/parallel/test-stream-iter-from-sync.js | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/internal/streams/iter/from.js b/lib/internal/streams/iter/from.js index ab786dc38258..e434ff9c253b 100644 --- a/lib/internal/streams/iter/from.js +++ b/lib/internal/streams/iter/from.js @@ -738,7 +738,7 @@ function fromSync(input) { if (!isIterable && isAsyncIterable(input)) { throw new ERR_INVALID_ARG_TYPE( 'input', - 'a synchronous input (not AsyncIterable)', + 'a synchronous input, not an async iterable', input, ); } @@ -748,7 +748,7 @@ function fromSync(input) { typeof input.then === 'function') { throw new ERR_INVALID_ARG_TYPE( 'input', - 'a synchronous input (not Promise)', + 'a synchronous input, not a promise', input, ); } diff --git a/test/parallel/test-stream-iter-from-sync.js b/test/parallel/test-stream-iter-from-sync.js index af7acfd4a546..133597914c45 100644 --- a/test/parallel/test-stream-iter-from-sync.js +++ b/test/parallel/test-stream-iter-from-sync.js @@ -183,7 +183,10 @@ function testFromSyncIgnoresAsyncStreamable() { // Explicit async iterable rejected function testFromSyncRejectsAsyncIterable() { async function* gen() { yield [new TextEncoder().encode('a')]; } - assert.throws(() => fromSync(gen()), { code: 'ERR_INVALID_ARG_TYPE' }); + assert.throws(() => fromSync(gen()), { + code: 'ERR_INVALID_ARG_TYPE', + message: /must be a synchronous input, not an async iterable\./, + }); } function testFromSyncPrefersIteratorForDualIterable() { @@ -212,8 +215,10 @@ function testFromSyncPrefersIteratorForThenableIterable() { // Promise rejected function testFromSyncRejectsPromise() { - assert.throws(() => fromSync(Promise.resolve('hello')), - { code: 'ERR_INVALID_ARG_TYPE' }); + assert.throws(() => fromSync(Promise.resolve('hello')), { + code: 'ERR_INVALID_ARG_TYPE', + message: /must be a synchronous input, not a promise\./, + }); } // DataView input should be converted to Uint8Array (zero-copy) From 0e1c99afae72017dde381405e5b1a0184f4b61c6 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 3 Oct 2026 15:13:47 +0000 Subject: [PATCH 81/98] doc: document stream/iter behaviors and Node.js extensions Document behaviors that were previously only visible in the code: - the writer "closing" state after end()/endSync(), - writes made from argument conversion being ordered first, - zero-length push() writes not being buffered, - broadcast.cancel() also closing the paired writer, - merge() not waiting for the other sources' cleanup on error, - the options argument passed to the tap() callback, - FileHandle writer() ordering of un-awaited writes, and the lazy locking of FileHandle pull() and pullSync(). Also link the WinterTC Iterable Streams API draft and list the exports that are Node.js extensions to it. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/fs.md | 14 ++++++++---- doc/api/stream_iter.md | 48 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/doc/api/fs.md b/doc/api/fs.md index b4af15dae3f4..c8b2f1d2f99b 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -410,8 +410,9 @@ Return the file contents as an async iterable using the chunks (default 128 KB). If transforms are provided, they are applied via [`stream/iter pull()`][]. -The file handle is locked while the iterable is being consumed and unlocked -when iteration completes, an error occurs, or the consumer breaks. +The file handle is locked from the first read of the iterable, and unlocked +when iteration completes, an error occurs, or the consumer breaks. An iterable +that is never read does not lock the file handle. This function is only available when the `--experimental-stream-iter` flag is enabled. @@ -483,8 +484,8 @@ Synchronous counterpart of [`filehandle.pull()`][]. Returns a sync iterable that reads the file using synchronous I/O on the main thread. Reads are performed in `chunkSize`-byte chunks (default 128 KB). -The file handle is locked while the iterable is being consumed. Unlike the -async `pull()`, this method does not support `AbortSignal` since all +The file handle is locked from the first read of the iterable until iteration +ends, as with [`filehandle.pull()`][]. Unlike the async `pull()`, this method does not support `AbortSignal` since all operations are synchronous. This function is only available when the `--experimental-stream-iter` flag is @@ -1124,6 +1125,11 @@ The writer supports both `Symbol.asyncDispose` and `Symbol.dispose`: for it to complete. * `using w = fh.writer()` — calls `fail()` unconditionally. +Async writes (`write()` and `writev()`) that are started without awaiting the +previous one are performed one at a time, in the order they were called, so +they never overlap in the file. A queued write is not performed if the writer +fails, or its `signal` aborts, before its turn. + The `writeSync()` and `writevSync()` methods enable the try-sync fast path used by [`stream/iter pipeTo()`][]. When the reader's chunk size matches the writer's `chunkSize`, all writes in a `pipeTo()` pipeline complete diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index 9f28707f7105..421a9e2f1906 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -18,6 +18,13 @@ functions or objects with a `transform` method. Data flows in **batches** ({Uint8Array\[]} per iteration) to amortize the cost of async operations. +The module implements the WinterTC [Iterable Streams API][] draft. The +classic stream interop functions ([`fromReadable()`][], [`fromWritable()`][], +[`toReadable()`][], [`toReadableSync()`][] and [`toWritable()`][]), +[`Broadcast.from()`][], [`Share.from()`][], [`SyncShare.fromSync()`][] and the +protocol symbols exported by `Stream` are Node.js extensions that are not part +of the draft. + ```mjs import { from, pull, text } from 'node:stream/iter'; import { compressGzip, decompressGzip } from 'node:zlib/iter'; @@ -407,6 +414,18 @@ converted to a `USVString` and then UTF-8 encoded. `writev()` and chunks. Writer option dictionaries treat `null` as an empty dictionary and ignore unknown members. +Arguments are converted before the write itself starts. If the conversion runs +user code (for example a `toString()` method, or the iterator of a `writev()` +argument) that writes to the same writer, those writes are ordered before the +write whose argument is being converted, and they count against the same +backpressure limits. + +After `end()` or `endSync()` has been called, and until all buffered data has +been consumed, the writer is _closing_. While closing, `canWrite` is `null`, +`write()` and `writev()` reject with a `TypeError`, `writeSync()` and +`writevSync()` return `false`, `endSync()` returns `-1`, and calling `end()` +again returns the same promise as the first call. + Each async method has a synchronous `*Sync` counterpart designed for a try-fallback pattern: attempt the fast synchronous path first, and fall back to the async version only when the synchronous call indicates it could not @@ -855,6 +874,10 @@ run().catch(console.error); The writer returned by `push()` conforms to the \[Writer interface]\[]. +Zero-length chunks are accepted without being buffered: they are not delivered +to the consumer, and `writeSync()` and `write()` report success for them even +when backpressure is active. + ## Duplex channels ### `duplex([options])` @@ -1239,7 +1262,12 @@ added: v25.9.0 Merge multiple async iterables by yielding batches in temporal order (whichever source produces data first). All sources are consumed -concurrently. +concurrently, with at most one pending `next()` call per source. + +If a source fails, the returned iterable rejects with its error. `merge()` +calls `return()` on the other sources but does not wait for it to settle: an +async generator source that is suspended in an `await` only runs its cleanup +once that `await` completes. ```mjs import { from, merge, text } from 'node:stream/iter'; @@ -1265,8 +1293,9 @@ run().catch(console.error); added: v25.9.0 --> -* `callback` {Function} `(chunks) => void` Called with each batch and with - `null` when the source ends. +* `callback` {Function} `(chunks, options) => void` Called with each batch and + with `null` when the source ends. `options.signal` is the pipeline's + {AbortSignal}. * Returns: {Function} A stateless transform. Create a pass-through transform that observes batches without modifying them. @@ -1391,6 +1420,10 @@ run().catch(console.error); Cancel the broadcast. If `reason` is provided, all consumers reject with that exact reason. If it is omitted, consumers complete normally. +Cancelling also closes the paired writer: afterwards its `canWrite` is `null` +and `write()` rejects with a `TypeError`. This lets a [`Broadcast.from()`][] +pump stop pulling from its source. + #### `broadcast.consumerCount` * {number} @@ -2282,13 +2315,19 @@ const stream = fromSync(new Greeting('world')); console.log(textSync(stream)); // 'hello world' ``` +[Iterable Streams API]: https://iter-streams.proposal.wintertc.org/ [`--experimental-stream-iter`]: cli.md#--experimental-stream-iter +[`Broadcast.from()`]: #broadcastfrominput-options +[`Share.from()`]: #static-method-sharefrominput-options +[`SyncShare.fromSync()`]: #static-method-syncsharefromsyncinput-options [`array()`]: #arraysource-options [`arrayBuffer()`]: #arraybuffersource-options [`bytes()`]: #bytessource-options [`dump()`]: #drainsource-options [`from()`]: #frominput +[`fromReadable()`]: #fromreadablereadable [`fromSync()`]: #fromsyncinput +[`fromWritable()`]: #fromwritablewritable-options [`node:zlib/iter`]: zlib.md#iterable-compression [`ondrain()`]: #ondraindrainable [`pipeTo()`]: #pipetosource-transforms-writer-options @@ -2300,3 +2339,6 @@ console.log(textSync(stream)); // 'hello world' [`tap()`]: #tapcallback [`text()`]: #textsource-options [`toAsyncStreamable`]: #streamtoasyncstreamable +[`toReadable()`]: #toreadablesource-options +[`toReadableSync()`]: #toreadablesyncsource-options +[`toWritable()`]: #towritablewriter From 6cecf140e7cd4e57badd38f1a866533b807cf9d7 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sun, 4 Oct 2026 01:04:01 +0000 Subject: [PATCH 82/98] stream: do not fail the writer when pipeToSync() cannot close After writing every chunk, pipeToSync() treated endSync() returning -1 like any other error: it threw ERR_INVALID_STATE and, unless preventFail was set, called writer.fail() with it. -1 only means that the writer cannot close synchronously, e.g. a push() writer whose consumer has not drained it yet. All of the data had been accepted, but failing the writer discarded it, so the consumer saw an error instead of the end of the stream, and the caller could not recover. pipeToSync() still throws ERR_INVALID_STATE in that case, since it never falls back to the async end(), but it no longer fails the writer. The caller can still close it, e.g. with `await writer.end()`. Assisted-by: OpenCode Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/66483 Reviewed-By: Trivikram Kamat --- doc/api/stream_iter.md | 7 +++++ lib/internal/streams/iter/pull.js | 15 +++++++--- test/parallel/test-stream-iter-pipeto-edge.js | 28 ++++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/doc/api/stream_iter.md b/doc/api/stream_iter.md index 421a9e2f1906..bcc610045a0d 100644 --- a/doc/api/stream_iter.md +++ b/doc/api/stream_iter.md @@ -718,6 +718,13 @@ Synchronous version of [`pipeTo()`][]. The `source`, all transforms, and the The `writer` must have the `*Sync` methods (`writeSync`, `writevSync`, `endSync`) and `fail()` for this to work. +`pipeToSync()` never falls back to the asynchronous writer methods. If +`writer.endSync()` returns `-1` because the writer cannot close synchronously +(for example, a `push()` writer whose consumer has not read all of the data +yet), `pipeToSync()` throws `ERR_INVALID_STATE`. All of the data was accepted +by then, so the writer is not failed: it can still be closed, for example with +`await writer.end()`. + ### `pull(source[, ...transforms][, options])` - // <---- dist ---------> - // --------------[------------------+-------------------]-------------- - // M- w M+ - // - // ten_k - // <------> - // <---- rest ----> - // --------------[------------------+----+--------------]-------------- - // w V - // = buf * 10^k - // - // ten_k represents a unit-in-the-last-place in the decimal representation - // stored in buf. - // Decrement buf by ten_k while this takes buf closer to w. - - // The tests are written in this order to avoid overflow in unsigned - // integer arithmetic. +inline compute_mul_parity_result compute_mul_parity(std::uint64_t two_f, + uint128 c, int beta) noexcept { + // beta is always in [1, 63] here. + const uint128 r = umul192_lower128(two_f, c); + return {((r.high >> (64 - beta)) & 1) != 0, + ((r.high << beta) | (r.low >> (64 - beta))) == 0}; +} - while (rest < dist && delta - rest >= ten_k && - (rest + ten_k < dist || dist - rest > rest + ten_k - dist)) { - buf[len - 1]--; - rest += ten_k; - } +inline std::uint64_t +compute_left_endpoint_for_shorter_interval_case(uint128 c, int beta) noexcept { + return (c.high - (c.high >> (significand_bits + 2))) >> + (total_bits - significand_bits - 1 - beta); } -/*! -Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. -M- and M+ must be normalized and share the same exponent -60 <= e <= -32. -*/ -inline void grisu2_digit_gen(char *buffer, int &length, int &decimal_exponent, - diyfp M_minus, diyfp w, diyfp M_plus) { - static_assert(kAlpha >= -60, "internal error"); - static_assert(kGamma <= -32, "internal error"); +inline std::uint64_t +compute_right_endpoint_for_shorter_interval_case(uint128 c, int beta) noexcept { + return (c.high + (c.high >> (significand_bits + 1))) >> + (total_bits - significand_bits - 1 - beta); +} - // Generates the digits (and the exponent) of a decimal floating-point - // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's - // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= - // gamma. - // - // <--------------------------- delta ----> - // <---- dist ---------> - // --------------[------------------+-------------------]-------------- - // M- w M+ - // - // Grisu2 generates the digits of M+ from left to right and stops as soon as - // V is in [M-,M+]. +inline std::uint64_t +compute_round_up_for_shorter_interval_case(uint128 c, int beta) noexcept { + return ((c.high >> (total_bits - significand_bits - 2 - beta)) + 1) / 2; +} - std::uint64_t delta = - diyfp::sub(M_plus, M_minus) - .f; // (significand of (M+ - M-), implicit exponent is e) - std::uint64_t dist = - diyfp::sub(M_plus, w) - .f; // (significand of (M+ - w ), implicit exponent is e) +// floor(n / 10) for the shorter-interval right endpoint (n bounded so the +// single multiply below is exact). +inline std::uint64_t divide_by_pow10_1(std::uint64_t n) noexcept { + return umul128_upper64(n, std::uint64_t(1844674407370955162ull)); +} - // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): - // - // M+ = f * 2^e - // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e - // = ((p1 ) * 2^-e + (p2 )) * 2^e - // = p1 + p2 * 2^e +// floor(n / 1000) for the larger-divisor step (n bounded as above). +inline std::uint64_t divide_by_pow10_3(std::uint64_t n) noexcept { + return umul128_upper64(n, std::uint64_t(4722366482869645214ull)) >> 8; +} - const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e); +// Returns whether n is divisible by 10^kappa (= 100) and divides n by it. +inline bool check_divisibility_and_divide_by_pow10_kappa(std::uint64_t &n) noexcept { + // magic number for division by 100 (kappa == 2). + const std::uint32_t prod = std::uint32_t(n) * std::uint32_t(656); + const bool result = (prod & 0xffffu) < 656u; + n = std::uint64_t(prod >> 16); + return result; +} - auto p1 = static_cast( - M_plus.f >> - -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) - std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e +// Strip trailing decimal zeros from significand, bumping exponent accordingly. +// Branchless search; constants from the Dragonbox reference. +inline void remove_trailing_zeros(std::uint64_t &significand, int &exponent) noexcept { + std::uint64_t r = rotr64(significand * std::uint64_t(28999941890838049ull), 8); + bool b = r < std::uint64_t(184467440738ull); + int s = b ? 1 : 0; + significand = b ? r : significand; - // 1) - // - // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + r = rotr64(significand * std::uint64_t(182622766329724561ull), 4); + b = r < std::uint64_t(1844674407370956ull); + s = s * 2 + (b ? 1 : 0); + significand = b ? r : significand; - std::uint32_t pow10; - const int k = find_largest_pow10(p1, pow10); + r = rotr64(significand * std::uint64_t(10330176681277348905ull), 2); + b = r < std::uint64_t(184467440737095517ull); + s = s * 2 + (b ? 1 : 0); + significand = b ? r : significand; - // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) - // - // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) - // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) - // - // M+ = p1 + p2 * 2^e - // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e - // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e - // = d[k-1] * 10^(k-1) + ( rest) * 2^e - // - // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) - // - // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] - // - // but stop as soon as - // - // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + r = rotr64(significand * std::uint64_t(14757395258967641293ull), 1); + b = r < std::uint64_t(1844674407370955162ull); + s = s * 2 + (b ? 1 : 0); + significand = b ? r : significand; - int n = k; - while (n > 0) { - // Invariants: - // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) - // pow10 = 10^(n-1) <= p1 < 10^n - // - const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) - const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) - // - // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e - // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) - // - buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d - // - // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) - // - p1 = r; - n--; - // - // M+ = buffer * 10^n + (p1 + p2 * 2^e) - // pow10 = 10^n - // + exponent += s; +} - // Now check if enough digits have been generated. - // Compute - // - // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e - // - // Note: - // Since rest and delta share the same exponent e, it suffices to - // compare the significands. - const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2; - if (rest <= delta) { - // V = buffer * 10^n, with M- <= V <= M+. +// Dragonbox core: shortest (significand, exponent) such that +// value == significand * 10^exponent +// for a finite, positive, non-zero binary64 value, decomposed into its raw +// significand bits and biased exponent bits. +struct decimal_fp { + std::uint64_t significand; + int exponent; +}; - decimal_exponent += n; +inline decimal_fp to_decimal(std::uint64_t binary_significand, + int binary_exponent) noexcept { + const bool is_even = (binary_significand % 2 == 0); + std::uint64_t two_fc = binary_significand * 2; + + // Is the input a normal number? + if (binary_exponent != 0) { + binary_exponent += exponent_bias - significand_bits; + + // Shorter interval case; proceed like Schubfach. + if (two_fc == 0) { + const int minus_k = + floor_log10_pow2_minus_log10_4_over_3(binary_exponent); + const int beta = binary_exponent + floor_log2_pow10(-minus_k); + const uint128 c = cache[-minus_k - cache_min_k]; + + std::uint64_t xi = + compute_left_endpoint_for_shorter_interval_case(c, beta); + const std::uint64_t zi = + compute_right_endpoint_for_shorter_interval_case(c, beta); + + // If the left endpoint is not an integer, increase it. (Both endpoints + // are always included since the significand is even.) + if (!(binary_exponent >= + case_shorter_interval_left_endpoint_lower_threshold && + binary_exponent <= + case_shorter_interval_left_endpoint_upper_threshold)) { + ++xi; + } - // We may now just stop. But instead look if the buffer could be - // decremented to bring V closer to w. - // - // pow10 = 10^n is now 1 ulp in the decimal representation V. - // The rounding procedure works with diyfp's with an implicit - // exponent of e. - // - // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e - // - const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e; - grisu2_round(buffer, length, dist, delta, rest, ten_n); + // Try the bigger divisor. + std::uint64_t decimal_significand = divide_by_pow10_1(zi); + if (decimal_significand * 10 >= xi) { + int decimal_exponent = minus_k + 1; + remove_trailing_zeros(decimal_significand, decimal_exponent); + return {decimal_significand, decimal_exponent}; + } - return; + // Otherwise, compute the round-up of y. + decimal_significand = + compute_round_up_for_shorter_interval_case(c, beta); + // On a tie, choose the even one. + if ((decimal_significand % 2 != 0) && + binary_exponent >= shorter_interval_tie_lower_threshold && + binary_exponent <= shorter_interval_tie_upper_threshold) { + --decimal_significand; + } else if (decimal_significand < xi) { + ++decimal_significand; + } + return {decimal_significand, minus_k}; } - pow10 /= 10; - // - // pow10 = 10^(n-1) <= p1 < 10^n - // Invariants restored. - } - - // 2) - // - // The digits of the integral part have been generated: - // - // M+ = d[k-1]...d[1]d[0] + p2 * 2^e - // = buffer + p2 * 2^e - // - // Now generate the digits of the fractional part p2 * 2^e. - // - // Note: - // No decimal point is generated: the exponent is adjusted instead. - // - // p2 actually represents the fraction - // - // p2 * 2^e - // = p2 / 2^-e - // = d[-1] / 10^1 + d[-2] / 10^2 + ... - // - // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) - // - // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m - // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) - // - // using - // - // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) - // = ( d) * 2^-e + ( r) - // - // or - // 10^m * p2 * 2^e = d + r * 2^e - // - // i.e. - // - // M+ = buffer + p2 * 2^e - // = buffer + 10^-m * (d + r * 2^e) - // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e - // - // and stop as soon as 10^-m * r * 2^e <= delta * 2^e - - int m = 0; - for (;;) { - // Invariant: - // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) - // * 2^e - // = buffer * 10^-m + 10^-m * (p2 ) - // * 2^e = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e = - // buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + - // (10*p2 mod 2^-e)) * 2^e - // - p2 *= 10; - const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e - const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e - // - // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e - // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) - // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e - // - buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d - // - // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e - // - p2 = r; - m++; - // - // M+ = buffer * 10^-m + 10^-m * p2 * 2^e - // Invariant restored. - - // Check if enough digits have been generated. - // - // 10^-m * p2 * 2^e <= delta * 2^e - // p2 * 2^e <= 10^m * delta * 2^e - // p2 <= 10^m * delta - delta *= 10; - dist *= 10; - if (p2 <= delta) { + // Normal interval case. + two_fc |= (std::uint64_t(1) << (significand_bits + 1)); + } else { + // Subnormal number: normal interval case. + binary_exponent = min_exponent - significand_bits; + } + + // Step 1: Schubfach multiplier calculation. + const int minus_k = floor_log10_pow2(binary_exponent) - kappa; + const uint128 c = cache[-minus_k - cache_min_k]; + const int beta = binary_exponent + floor_log2_pow10(-minus_k); + + const std::uint64_t deltai = compute_delta(c, beta); + const compute_mul_result z_result = + compute_mul((two_fc | 1) << beta, c); + + // Step 2: Try larger divisor; remove trailing zeros if necessary. + std::uint64_t decimal_significand = divide_by_pow10_3(z_result.integer_part); + std::uint64_t r = + z_result.integer_part - std::uint64_t(big_divisor) * decimal_significand; + + do { + if (r < deltai) { + // Exclude the right endpoint if necessary. + if ((r | std::uint64_t(!z_result.is_integer) | std::uint64_t(is_even)) == + 0) { + --decimal_significand; + r = big_divisor; + break; + } + } else if (r > deltai) { break; + } else { + // r == deltai; compare fractional parts. + const compute_mul_parity_result x_result = + compute_mul_parity(two_fc - 1, c, beta); + if (!(x_result.parity | (x_result.is_integer & is_even))) { + break; + } } - } - - // V = buffer * 10^-m, with M- <= V <= M+. - - decimal_exponent -= m; - - // 1 ulp in the decimal representation is now 10^-m. - // Since delta and dist are now scaled by 10^m, we need to do the - // same with ulp in order to keep the units in sync. - // - // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e - // - const std::uint64_t ten_m = one.f; - grisu2_round(buffer, length, dist, delta, p2, ten_m); - // By construction this algorithm generates the shortest possible decimal - // number (Loitsch, Theorem 6.2) which rounds back to w. - // For an input number of precision p, at least - // - // N = 1 + ceil(p * log_10(2)) - // - // decimal digits are sufficient to identify all binary floating-point - // numbers (Matula, "In-and-Out conversions"). - // This implies that the algorithm does not produce more than N decimal - // digits. - // - // N = 17 for p = 53 (IEEE double precision) - // N = 9 for p = 24 (IEEE single precision) -} + int decimal_exponent = minus_k + kappa + 1; + remove_trailing_zeros(decimal_significand, decimal_exponent); + return {decimal_significand, decimal_exponent}; + } while (false); -/*! -v = buf * 10^decimal_exponent -len is the length of the buffer (number of decimal digits) -The buffer must be large enough, i.e. >= max_digits10. -*/ -inline void grisu2(char *buf, int &len, int &decimal_exponent, diyfp m_minus, - diyfp v, diyfp m_plus) { - - // --------(-----------------------+-----------------------)-------- (A) - // m- v m+ - // - // --------------------(-----------+-----------------------)-------- (B) - // m- v m+ - // - // First scale v (and m- and m+) such that the exponent is in the range - // [alpha, gamma]. + // Step 3: Find the significand with the smaller divisor. + decimal_significand *= 10; - const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + std::uint64_t dist = r - (deltai / 2) + (small_divisor / 2); + const bool approx_y_parity = ((dist ^ (small_divisor / 2)) & 1) != 0; - const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + const bool divisible_by_small_divisor = + check_divisibility_and_divide_by_pow10_kappa(dist); - // The exponent of the products is = v.e + c_minus_k.e + q and is in the range - // [alpha,gamma] - const diyfp w = diyfp::mul(v, c_minus_k); - const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); - const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + decimal_significand += dist; - // ----(---+---)---------------(---+---)---------------(---+---)---- - // w- w w+ - // = c*m- = c*v = c*m+ - // - // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and - // w+ are now off by a small amount. - // In fact: - // - // w - v * 10^k < 1 ulp - // - // To account for this inaccuracy, add resp. subtract 1 ulp. - // - // --------+---[---------------(---+---)---------------]---+-------- - // w- M- w M+ w+ - // - // Now any number in [M-, M+] (bounds included) will round to w when input, - // regardless of how the input rounding algorithm breaks ties. - // - // And digit_gen generates the shortest possible such number in [M-, M+]. - // Note that this does not mean that Grisu2 always generates the shortest - // possible number in the interval (m-, m+). - const diyfp M_minus(w_minus.f + 1, w_minus.e); - const diyfp M_plus(w_plus.f - 1, w_plus.e); - - decimal_exponent = -cached.k; // = -(-k) = k + if (divisible_by_small_divisor) { + const compute_mul_parity_result y_result = + compute_mul_parity(two_fc, c, beta); + if (y_result.parity != approx_y_parity) { + --decimal_significand; + } else if ((decimal_significand % 2) != 0 && y_result.is_integer) { + // On a tie (y is an integer), choose the even one. + --decimal_significand; + } + } - grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); + return {decimal_significand, minus_k + kappa}; } /*! -v = buf * 10^decimal_exponent -len is the length of the buffer (number of decimal digits) -The buffer must be large enough, i.e. >= max_digits10. +Fills buf with the shortest decimal digits of 'value' (which must be finite, +positive and non-zero), sets len to the number of digits, and decimal_exponent +so that value == (buf, interpreted as an integer) * 10^decimal_exponent. +This mirrors the contract of the previous grisu2() entry point. */ -template -void grisu2(char *buf, int &len, int &decimal_exponent, FloatType value) { - static_assert(diyfp::kPrecision >= std::numeric_limits::digits + 3, - "internal error: not enough precision"); - - // If the neighbors (and boundaries) of 'value' are always computed for - // double-precision numbers, all float's can be recovered using strtod (and - // strtof). However, the resulting decimal representations are not exactly - // "short". - // - // The documentation for 'std::to_chars' - // (https://en.cppreference.com/w/cpp/utility/to_chars) says "value is - // converted to a string as if by std::sprintf in the default ("C") locale" - // and since sprintf promotes float's to double's, I think this is exactly - // what 'std::to_chars' does. On the other hand, the documentation for - // 'std::to_chars' requires that "parsing the representation using the - // corresponding std::from_chars function recovers value exactly". That - // indicates that single precision floating-point numbers should be recovered - // using 'std::strtof'. - // - // NB: If the neighbors are computed for single-precision numbers, there is a - // single float - // (7.0385307e-26f) which can't be recovered using strtod. The resulting - // double precision value is off by 1 ulp. -#if 0 - const boundaries w = compute_boundaries(static_cast(value)); -#else - const boundaries w = compute_boundaries(value); -#endif - - grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +inline void dragonbox(char *buf, int &len, int &decimal_exponent, + double value) { + std::uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + const std::uint64_t binary_significand = + bits & ((std::uint64_t(1) << significand_bits) - 1); + const int binary_exponent = + int((bits >> significand_bits) & 0x7ff); + + const decimal_fp dec = to_decimal(binary_significand, binary_exponent); + + // Convert the decimal significand to digits: + // 1) Proceed 2 digits at a time (s % 100) via a 00..99 lookup table + // (see Alexandrescu, "Three Optimization Tips for C++", 2012), + // 2) Digits come out least-significant first, writing them back-to-front + // with p = tmp + sizeof(tmp); to avoid reversal pass + // 3) Proceed remaining digits after loop to avoid branchs inside it + // 4) memcpy digits to char* buf (inside function input) + static const char digits2[201] = + "0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"; + // Digit area is 24; +16 padding lets us always memcpy 16 (+1) bytes with a + // compile-time size so the compiler inlines (no libc size-class branches). + // Callers must provide to_chars_buffer_size (40) bytes for the same reason. + constexpr int digit_area = 24; + char tmp[digit_area + 16]; + char *p = tmp + digit_area; // write backward + std::uint64_t s = dec.significand; + while (s >= 100) { + const std::uint32_t idx = static_cast(s % 100) * 2; + s /= 100; + p -= 2; + p[0] = digits2[idx]; + p[1] = digits2[idx + 1]; + } + if (s >= 10) { + const std::uint32_t idx = static_cast(s) * 2; + p -= 2; + p[0] = digits2[idx]; + p[1] = digits2[idx + 1]; + } else { + *--p = static_cast('0' + s); + } + // Fixed-size copy: double has at most 17 significant digits. + std::memcpy(buf, p, 16); + buf[16] = p[16]; + len = static_cast(tmp + digit_area - p); + decimal_exponent = dec.exponent; } /*! @@ -4111,11 +4625,16 @@ inline char *format_buffer(char *buf, int len, int decimal_exponent, // k is the length of the buffer (number of decimal digits) // n is the position of the decimal point relative to the start of the buffer. + // All mem* sizes below are compile-time constants so the compiler inlines + // them as plain loads/stores. That requires over-writing past the logical + // string length; callers must reserve to_chars_buffer_size (40) bytes. + // Logical output is still bounded by ~24 characters; only the returned + // pointer reflects the true length. + if (k <= n && n <= max_exp) { // digits[000] // len <= max_exp + 2 - - std::memset(buf + k, '0', static_cast(n) - static_cast(k)); + std::memset(buf + k, '0', 16); // Make it look like a floating-point number (#362, #378) buf[n + 0] = '.'; buf[n + 1] = '0'; @@ -4125,690 +4644,5307 @@ inline char *format_buffer(char *buf, int len, int decimal_exponent, if (0 < n && n <= max_exp) { // dig.its // len <= max_digits10 + 1 - std::memmove(buf + (static_cast(n) + 1), buf + n, - static_cast(k) - static_cast(n)); + // Shift the fractional digits one place right via a temp (overlap). + char shifted[16]; + std::memcpy(shifted, buf + static_cast(n), 16); + std::memcpy(buf + (static_cast(n) + 1), shifted, 16); buf[n] = '.'; return buf + (static_cast(k) + 1U); } - if (min_exp < n && n <= 0) { - // 0.[000]digits - // len <= 2 + (-min_exp - 1) + max_digits10 + if (min_exp < n && n <= 0) { + // 0.[000]digits + // With kMinExp = -4, n is in {-3,-2,-1,0}, so pad = -n is 0..3. + // len <= 2 + (-min_exp - 1) + max_digits10 + char digits[17]; + std::memcpy(digits, buf, 17); + const size_t pad = static_cast(-n); // 0..3 + buf[0] = '0'; + buf[1] = '.'; + // Fixed upper bound on leading zeros; only the first `pad` matter. + std::memset(buf + 2, '0', 4); + std::memcpy(buf + 2 + pad, digits, 17); + return buf + (2U + pad + static_cast(k)); + } + + if (k == 1) { + // dE+123 + // len <= 1 + 5 + buf += 1; + } else { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + // k-1 <= 16 for double; fixed-size shift via temp (overlap). + char shifted[16]; + std::memcpy(shifted, buf + 1, 16); + std::memcpy(buf + 2, shifted, 16); + buf[1] = '.'; + buf += 1 + static_cast(k); + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // NS dtoa_impl + +/*! +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must have at least to_chars_buffer_size (40) writable bytes. + Only ~24 characters are ever part of the logical result, but fixed-size + 16/17-byte mem* over-writes require the extra scratch for safety. +@note The result is NOT null-terminated. +*/ +char *to_chars(char *first, const char *last, double value) { + static_cast(last); // maybe unused - fix warning + bool negative = std::signbit(value); + if (negative) { + value = -value; + *first++ = '-'; + } + + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::dragonbox(first, len, decimal_exponent, value); + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + constexpr int kMaxExp = std::numeric_limits::digits10; + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, + kMaxExp); +} +} // NS internal +} // NS simdjson + +#endif // SIMDJSON_SRC_TO_CHARS_CPP + + +/* end file to_chars.cpp */ +/* including from_chars.cpp: #include */ +/* begin file from_chars.cpp */ +#ifndef SIMDJSON_SRC_FROM_CHARS_CPP +#define SIMDJSON_SRC_FROM_CHARS_CPP + +/* skipped duplicate #include */ + +/* including simdjson/internal/fast_float.h: #include "simdjson/internal/fast_float.h" */ +/* begin file simdjson/internal/fast_float.h */ +// Vendored from fast_float v8.2.10, generated by tools/vendor_fast_float.sh. +// Do not edit by hand; re-run the script to update. +// +// https://github.com/fastfloat/fast_float +// Licensed under Apache-2.0 OR MIT OR BSL-1.0, at your option. +// +// simdjson uses this for two things that its own number parser cannot do: +// * the slow path for numbers with more than 19 significant digits, where +// fast_float's bigint comparison is several times quicker than the +// Wuffs-derived decimal shifting it replaced (see src/from_chars.cpp), and +// * correctly rounded parsing inside a constant expression, which the runtime +// path cannot offer because it relies on memcpy and __uint128_t (see +// compile_time_json-inl.h). +// +// Two edits are applied by the script. Every fast_float name is rewritten so +// that this copy cannot collide with a copy of fast_float that the surrounding +// program includes for itself: namespace fast_float -> simdjson_fast_float, +// FASTFLOAT_* -> SIMDJSON_FASTFLOAT_*, fastfloat_* -> simdjson_fastfloat_*. And +// the accented letters in the attribution comments below are folded to ASCII, +// to keep the tree ASCII-only; no disrespect to the people named is intended. +// simdjson_fast_float by Daniel Lemire +// simdjson_fast_float by Joao Paulo Magalhaes +// +// +// with contributions from Eugene Golushkov +// with contributions from Maksim Kita +// with contributions from Marcin Wojdyr +// with contributions from Neal Richardson +// with contributions from Tim Paine +// with contributions from Fabio Pellacini +// with contributions from Lenard Szolnoki +// with contributions from Jan Pharago +// with contributions from Maya Warrier +// with contributions from Taha Khokhar +// with contributions from Anders Dalvander +// +// +// Licensed under the Apache License, Version 2.0, or the +// MIT License or the Boost License. This file may not be copied, +// modified, or distributed except according to those terms. +// +// MIT License Notice +// +// MIT License +// +// Copyright (c) 2021 The simdjson_fast_float authors +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Apache License (Version 2.0) Notice +// +// Copyright 2021 The simdjson_fast_float authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// +// BOOST License Notice +// +// Boost Software License - Version 1.0 - August 17th, 2003 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +#ifndef SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H +#define SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H + +#ifdef __has_include +#if __has_include() +#include +#endif +#endif + +// Testing for https://wg21.link/N3652, adopted in C++14 +#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304 +#define SIMDJSON_FASTFLOAT_CONSTEXPR14 constexpr +#else +#define SIMDJSON_FASTFLOAT_CONSTEXPR14 +#endif + +#if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L +#define SIMDJSON_FASTFLOAT_HAS_BIT_CAST 1 +#else +#define SIMDJSON_FASTFLOAT_HAS_BIT_CAST 0 +#endif + +#if defined(__cpp_lib_is_constant_evaluated) && \ + __cpp_lib_is_constant_evaluated >= 201811L +#define SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 1 +#else +#define SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 0 +#endif + +#if defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606L +#define SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(x) if constexpr (x) +#else +#define SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(x) if (x) +#endif + +// Testing for relevant C++20 constexpr library features +#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && SIMDJSON_FASTFLOAT_HAS_BIT_CAST && \ + defined(__cpp_lib_constexpr_algorithms) && \ + __cpp_lib_constexpr_algorithms >= 201806L /*For std::copy and std::fill*/ +#define SIMDJSON_FASTFLOAT_CONSTEXPR20 constexpr +#define SIMDJSON_FASTFLOAT_IS_CONSTEXPR 1 +#else +#define SIMDJSON_FASTFLOAT_CONSTEXPR20 +#define SIMDJSON_FASTFLOAT_IS_CONSTEXPR 0 +#endif + +#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +#define SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 0 +#else +#define SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 1 +#endif + +#endif // SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H + +#ifndef SIMDJSON_FASTFLOAT_FLOAT_COMMON_H +#define SIMDJSON_FASTFLOAT_FLOAT_COMMON_H + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __has_include +#if __has_include() && (__cplusplus > 202002L || (defined(_MSVC_LANG) && (_MSVC_LANG > 202002L))) +#include +#endif +#endif + +#define SIMDJSON_FASTFLOAT_VERSION_MAJOR 8 +#define SIMDJSON_FASTFLOAT_VERSION_MINOR 2 +#define SIMDJSON_FASTFLOAT_VERSION_PATCH 10 + +#define SIMDJSON_FASTFLOAT_STRINGIZE_IMPL(x) #x +#define SIMDJSON_FASTFLOAT_STRINGIZE(x) SIMDJSON_FASTFLOAT_STRINGIZE_IMPL(x) + +#define SIMDJSON_FASTFLOAT_VERSION_STR \ + SIMDJSON_FASTFLOAT_STRINGIZE(SIMDJSON_FASTFLOAT_VERSION_MAJOR) \ + "." SIMDJSON_FASTFLOAT_STRINGIZE(SIMDJSON_FASTFLOAT_VERSION_MINOR) "." SIMDJSON_FASTFLOAT_STRINGIZE( \ + SIMDJSON_FASTFLOAT_VERSION_PATCH) + +#define SIMDJSON_FASTFLOAT_VERSION \ + (SIMDJSON_FASTFLOAT_VERSION_MAJOR * 10000 + SIMDJSON_FASTFLOAT_VERSION_MINOR * 100 + \ + SIMDJSON_FASTFLOAT_VERSION_PATCH) + +namespace simdjson_fast_float { + +enum class chars_format : uint64_t; + +namespace detail { +constexpr chars_format basic_json_fmt = chars_format(1 << 5); +constexpr chars_format basic_fortran_fmt = chars_format(1 << 6); +} // namespace detail + +enum class chars_format : uint64_t { + scientific = 1 << 0, + fixed = 1 << 2, + hex = 1 << 3, + no_infnan = 1 << 4, + // RFC 8259: https://datatracker.ietf.org/doc/html/rfc8259#section-6 + json = uint64_t(detail::basic_json_fmt) | fixed | scientific | no_infnan, + // Extension of RFC 8259 where, e.g., "inf" and "nan" are allowed. + json_or_infnan = uint64_t(detail::basic_json_fmt) | fixed | scientific, + fortran = uint64_t(detail::basic_fortran_fmt) | fixed | scientific, + general = fixed | scientific, + allow_leading_plus = 1 << 7, + skip_white_space = 1 << 8, +}; + +template struct from_chars_result_t { + UC const *ptr; + std::errc ec; + + // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2497r0.html + constexpr explicit operator bool() const noexcept { + return ec == std::errc(); + } +}; + +using from_chars_result = from_chars_result_t; + +template struct parse_options_t { + constexpr explicit parse_options_t(chars_format fmt = chars_format::general, + UC dot = UC('.'), int b = 10) + : format(fmt), decimal_point(dot), base(b) {} + + /** Which number formats are accepted */ + chars_format format; + /** The character used as decimal point */ + UC decimal_point; + /** The base used for integers */ + int base; +}; + +using parse_options = parse_options_t; + +} // namespace simdjson_fast_float + +#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST +#include +#endif + +#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(__amd64) || defined(__aarch64__) || defined(_M_ARM64) || \ + defined(__MINGW64__) || defined(__s390x__) || \ + (defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ + defined(__PPC64LE__)) || \ + defined(__loongarch64) || (defined(__riscv) && __riscv_xlen == 64)) +#define SIMDJSON_FASTFLOAT_64BIT 1 +#elif (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__arm__) || defined(_M_ARM) || defined(__ppc__) || \ + defined(__MINGW32__) || defined(__EMSCRIPTEN__) || \ + (defined(__riscv) && __riscv_xlen == 32)) +#define SIMDJSON_FASTFLOAT_32BIT 1 +#else + // Need to check incrementally, since SIZE_MAX is a size_t, avoid overflow. +// We can never tell the register width, but the SIZE_MAX is a good +// approximation. UINTPTR_MAX and INTPTR_MAX are optional, so avoid them for max +// portability. +#if SIZE_MAX == 0xffff +#error Unknown platform (16-bit, unsupported) +#elif SIZE_MAX == 0xffffffff +#define SIMDJSON_FASTFLOAT_32BIT 1 +#elif SIZE_MAX == 0xffffffffffffffff +#define SIMDJSON_FASTFLOAT_64BIT 1 +#else +#error Unknown platform (not 32-bit, not 64-bit?) +#endif +#endif + +#if ((defined(_WIN32) || defined(_WIN64)) && !defined(__clang__)) || \ + (defined(_M_ARM64) && !defined(__MINGW32__)) +#include +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define SIMDJSON_FASTFLOAT_VISUAL_STUDIO 1 +#endif + +#if defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined _WIN32 +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#else +#if defined(__APPLE__) || defined(__FreeBSD__) +#include +#elif defined(sun) || defined(__sun) +#include +#elif defined(__MVS__) +#include +#else +#ifdef __has_include +#if __has_include() +#include +#endif //__has_include() +#endif //__has_include +#endif +# +#ifndef __BYTE_ORDER__ +// safe choice +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#endif +# +#ifndef __ORDER_LITTLE_ENDIAN__ +// safe choice +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#endif +# +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#else +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 1 +#endif +#endif + +#if defined(__SSE2__) || (defined(SIMDJSON_FASTFLOAT_VISUAL_STUDIO) && \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2))) +#define SIMDJSON_FASTFLOAT_SSE2 1 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define SIMDJSON_FASTFLOAT_NEON 1 +#endif + +#if defined(SIMDJSON_FASTFLOAT_SSE2) || defined(SIMDJSON_FASTFLOAT_NEON) +#define SIMDJSON_FASTFLOAT_HAS_SIMD 1 +#endif + +#if defined(__GNUC__) +// disable -Wcast-align=strict (GCC only) +#define SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wcast-align\"") +#else +#define SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS +#endif + +#if defined(__GNUC__) +#define SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS _Pragma("GCC diagnostic pop") +#else +#define SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +#endif + +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#define simdjson_fastfloat_really_inline __forceinline +#else +#define simdjson_fastfloat_really_inline inline __attribute__((always_inline)) +#endif + +// Branch-probability hint marking the rare slow-path branches as cold, so the +// optimizer keeps the out-of-line slow-path re-parse off the hot path (and does +// not duplicate the force-inlined hot scanner into the caller, which bloated +// the hot frame and hurt ILP on some targets). Used at the call site as +// if simdjson_fastfloat_unlikely(cond) { ... } +// (the macro supplies the parentheses). It expands to the standard [[unlikely]] +// attribute when supported, otherwise to __builtin_expect on GCC/Clang, or +// to a no-op elsewhere (e.g. pre-C++20 MSVC, which has no equivalent hint). +#ifdef __has_cpp_attribute +#if __has_cpp_attribute(unlikely) >= 201803L +// g++-9 hits hits this branch, but then fails to compile +// [[unlikely]]. This happens only with g++-9. +#if !defined(__GNUC__) || (__GNUC__ != 9) +#define SIMDJSON_FASTFLOAT_USE_UNLIKELY_ATTR +#endif +#endif +#endif + +#ifdef SIMDJSON_FASTFLOAT_USE_UNLIKELY_ATTR +#define simdjson_fastfloat_unlikely(x) (x) [[unlikely]] +#elif defined(__GNUC__) || defined(__clang__) +#define simdjson_fastfloat_unlikely(x) (__builtin_expect(!!(x), 0)) +#else +#define simdjson_fastfloat_unlikely(x) (x) +#endif + +#ifndef SIMDJSON_FASTFLOAT_ASSERT +#define SIMDJSON_FASTFLOAT_ASSERT(x) \ + { \ + static_cast(x); \ + } +#endif + +#ifndef SIMDJSON_FASTFLOAT_DEBUG_ASSERT +#define SIMDJSON_FASTFLOAT_DEBUG_ASSERT(x) \ + { \ + static_cast(x); \ + } +#endif + +// rust style `try!()` macro, or `?` operator +#define SIMDJSON_FASTFLOAT_TRY(x) \ + { \ + if (!(x)) \ + return false; \ + } + +#define SIMDJSON_FASTFLOAT_ENABLE_IF(...) \ + typename std::enable_if<(__VA_ARGS__), int>::type + +namespace simdjson_fast_float { + +simdjson_fastfloat_really_inline constexpr bool cpp20_and_in_constexpr() { +#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED + return std::is_constant_evaluated(); +#else + return false; +#endif +} + +template +struct is_supported_float_type + : std::integral_constant< + bool, std::is_same::value || std::is_same::value +#ifdef __STDCPP_FLOAT64_T__ + || std::is_same::value +#endif +#ifdef __STDCPP_FLOAT32_T__ + || std::is_same::value +#endif +#ifdef __STDCPP_FLOAT16_T__ + || std::is_same::value +#endif +#ifdef __STDCPP_BFLOAT16_T__ + || std::is_same::value +#endif + > { +}; + +template +using equiv_uint_t = typename std::conditional< + sizeof(T) == 1, uint8_t, + typename std::conditional< + sizeof(T) == 2, uint16_t, + typename std::conditional::type>::type>::type; + +template struct is_supported_integer_type : std::is_integral {}; + +template +struct is_supported_char_type + : std::integral_constant::value || + std::is_same::value || + std::is_same::value || + std::is_same::value +#ifdef __cpp_char8_t + || std::is_same::value +#endif + > { +}; + +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool +simdjson_fastfloat_strncasecmp3(UC const *actual_mixedcase, + UC const *expected_lowercase) { + uint64_t mask{0}; + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { + mask = 0x0020002000200020; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + mask = 0x0000002000000020; + } + else { + return false; + } + + uint64_t val1{0}, val2{0}; + if (cpp20_and_in_constexpr()) { + for (size_t i = 0; i < 3; i++) { + if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { + return false; + } + } + return true; + } else { + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1 || sizeof(UC) == 2) { + ::memcpy(&val1, actual_mixedcase, 3 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 3 * sizeof(UC)); + val1 |= mask; + val2 |= mask; + return val1 == val2; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + return (actual_mixedcase[2] | 32) == (expected_lowercase[2]); + } + else { + return false; + } + } +} + +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool +simdjson_fastfloat_strncasecmp5(UC const *actual_mixedcase, + UC const *expected_lowercase) { + uint64_t mask{0}; + uint64_t val1{0}, val2{0}; + if (cpp20_and_in_constexpr()) { + for (size_t i = 0; i < 5; i++) { + if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { + return false; + } + } + return true; + } else { + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { + mask = 0x2020202020202020; + ::memcpy(&val1, actual_mixedcase, 5 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 5 * sizeof(UC)); + val1 |= mask; + val2 |= mask; + return val1 == val2; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { + mask = 0x0020002000200020; + ::memcpy(&val1, actual_mixedcase, 4 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 4 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + mask = 0x0000002000000020; + ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + ::memcpy(&val1, actual_mixedcase + 2, 2 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase + 2, 2 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); + } + else { + return false; + } + } +} + +// Compares two ASCII strings in a case insensitive manner. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool +simdjson_fastfloat_strncasecmp(UC const *actual_mixedcase, UC const *expected_lowercase, + size_t length) { + uint64_t mask{0}; + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { + mask = 0x0020002000200020; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + mask = 0x0000002000000020; + } + else { + return false; + } + + if (cpp20_and_in_constexpr()) { + for (size_t i = 0; i < length; i++) { + if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { + return false; + } + } + return true; + } else { + uint64_t val1{0}, val2{0}; + size_t sz{8 / (sizeof(UC))}; + for (size_t i = 0; i < length; i += sz) { + val1 = val2 = 0; + sz = sz < (length - i) ? sz : length - i; + ::memcpy(&val1, actual_mixedcase + i, sz * sizeof(UC)); + ::memcpy(&val2, expected_lowercase + i, sz * sizeof(UC)); + val1 |= mask; + val2 |= mask; + if (val1 != val2) { + return false; + } + } + return true; + } +} + +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif + +// a pointer and a length to a contiguous block of memory +template struct span { + T const *ptr; + size_t length; + + constexpr span(T const *_ptr, size_t _length) : ptr(_ptr), length(_length) {} + + constexpr span() : ptr(nullptr), length(0) {} + + constexpr size_t len() const noexcept { return length; } + + SIMDJSON_FASTFLOAT_CONSTEXPR14 const T &operator[](size_t index) const noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + return ptr[index]; + } +}; + +struct value128 { + uint64_t low; + uint64_t high; + + constexpr value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {} + + constexpr value128() : low(0), high(0) {} +}; + +/* Helper C++14 constexpr generic implementation of leading_zeroes */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int +leading_zeroes_generic(uint64_t input_num, int last_bit = 0) { + if (input_num & uint64_t(0xffffffff00000000)) { + input_num >>= 32; + last_bit |= 32; + } + if (input_num & uint64_t(0xffff0000)) { + input_num >>= 16; + last_bit |= 16; + } + if (input_num & uint64_t(0xff00)) { + input_num >>= 8; + last_bit |= 8; + } + if (input_num & uint64_t(0xf0)) { + input_num >>= 4; + last_bit |= 4; + } + if (input_num & uint64_t(0xc)) { + input_num >>= 2; + last_bit |= 2; + } + if (input_num & uint64_t(0x2)) { /* input_num >>= 1; */ + last_bit |= 1; + } + return 63 - last_bit; +} + +/* result might be undefined when input_num is zero */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 int +leading_zeroes(uint64_t input_num) { + assert(input_num > 0); + if (cpp20_and_in_constexpr()) { + return leading_zeroes_generic(input_num); + } +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#if defined(_M_X64) || defined(_M_ARM64) + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + _BitScanReverse64(&leading_zero, input_num); + return static_cast(63 - leading_zero); +#else + return leading_zeroes_generic(input_num); +#endif +#else + return __builtin_clzll(input_num); +#endif +} + +/* Helper C++14 constexpr generic implementation of countr_zero for 32-bit */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int +countr_zero_generic_32(uint32_t input_num) { + if (input_num == 0) { + return 32; + } + int last_bit = 0; + if (!(input_num & 0x0000FFFF)) { + input_num >>= 16; + last_bit |= 16; + } + if (!(input_num & 0x00FF)) { + input_num >>= 8; + last_bit |= 8; + } + if (!(input_num & 0x0F)) { + input_num >>= 4; + last_bit |= 4; + } + if (!(input_num & 0x3)) { + input_num >>= 2; + last_bit |= 2; + } + if (!(input_num & 0x1)) { + last_bit |= 1; + } + return last_bit; +} + +/* count trailing zeroes for 32-bit integers */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 int +countr_zero_32(uint32_t input_num) { + if (cpp20_and_in_constexpr()) { + return countr_zero_generic_32(input_num); + } +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO + unsigned long trailing_zero = 0; + if (_BitScanForward(&trailing_zero, input_num)) { + return static_cast(trailing_zero); + } + return 32; +#else + return input_num == 0 ? 32 : __builtin_ctz(input_num); +#endif +} + +// slow emulation routine for 32-bit +simdjson_fastfloat_really_inline constexpr uint64_t emulu(uint32_t x, uint32_t y) { + return x * static_cast(y); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t +umul128_generic(uint64_t ab, uint64_t cd, uint64_t *hi) { + uint64_t ad = + emulu(static_cast(ab >> 32), static_cast(cd)); + uint64_t bd = emulu(static_cast(ab), static_cast(cd)); + uint64_t adbc = + ad + emulu(static_cast(ab), static_cast(cd >> 32)); + uint64_t adbc_carry = static_cast(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = + emulu(static_cast(ab >> 32), static_cast(cd >> 32)) + + (adbc >> 32) + (adbc_carry << 32) + static_cast(lo < bd); + return lo; +} + +#ifdef SIMDJSON_FASTFLOAT_32BIT + +// slow emulation routine for 32-bit +#if !defined(__MINGW64__) +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t _umul128(uint64_t ab, + uint64_t cd, + uint64_t *hi) { + return umul128_generic(ab, cd, hi); +} +#endif // !__MINGW64__ + +#endif // SIMDJSON_FASTFLOAT_32BIT + +// compute 64-bit a*b +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 value128 +full_multiplication(uint64_t a, uint64_t b) { + if (cpp20_and_in_constexpr()) { + value128 answer; + answer.low = umul128_generic(a, b, &answer.high); + return answer; + } + value128 answer; +#if defined(_M_ARM64) && !defined(__MINGW32__) + // ARM64 has native support for 64-bit multiplications, no need to emulate + // But MinGW on ARM64 doesn't have native support for 64-bit multiplications + answer.high = __umulh(a, b); + answer.low = a * b; +#elif defined(SIMDJSON_FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__) && \ + !defined(_M_ARM64) && !defined(__GNUC__)) + answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64 +#elif defined(SIMDJSON_FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__) + __uint128_t r = static_cast<__uint128_t>(a) * b; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#else + answer.low = umul128_generic(a, b, &answer.high); +#endif + return answer; +} + +struct adjusted_mantissa { + uint64_t mantissa{0}; + int32_t power2{0}; // a negative value indicates an invalid result + adjusted_mantissa() = default; + + constexpr bool operator==(adjusted_mantissa const &o) const { + return mantissa == o.mantissa && power2 == o.power2; + } + + constexpr bool operator!=(adjusted_mantissa const &o) const { + return mantissa != o.mantissa || power2 != o.power2; + } +}; + +// Bias so we can get the real exponent with an invalid adjusted_mantissa. +constexpr static int32_t invalid_am_bias = -0x8000; + +// used for binary_format_lookup_tables::max_mantissa +constexpr uint64_t constant_55555 = 5 * 5 * 5 * 5 * 5; + +template struct binary_format_lookup_tables; + +template struct binary_format : binary_format_lookup_tables { + using equiv_uint = equiv_uint_t; + + static constexpr int mantissa_explicit_bits(); + static constexpr int minimum_exponent(); + static constexpr int infinite_power(); + static constexpr int sign_index(); + static constexpr int + min_exponent_fast_path(); // used when fegetround() == FE_TONEAREST + static constexpr int max_exponent_fast_path(); + static constexpr int max_exponent_round_to_even(); + static constexpr int min_exponent_round_to_even(); + static constexpr uint64_t max_mantissa_fast_path(int64_t power); + static constexpr uint64_t + max_mantissa_fast_path(); // used when fegetround() == FE_TONEAREST + static constexpr int largest_power_of_ten(); + static constexpr int smallest_power_of_ten(); + static constexpr T exact_power_of_ten(int64_t power); + static constexpr size_t max_digits(); + static constexpr equiv_uint exponent_mask(); + static constexpr equiv_uint mantissa_mask(); + static constexpr equiv_uint hidden_bit_mask(); +}; + +template struct binary_format_lookup_tables { + static constexpr double powers_of_ten[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; + + // Largest integer value v so that (5**index * v) <= 1<<53. + // 0x20000000000000 == 1 << 53 + static constexpr uint64_t max_mantissa[] = { + 0x20000000000000, + 0x20000000000000 / 5, + 0x20000000000000 / (5 * 5), + 0x20000000000000 / (5 * 5 * 5), + 0x20000000000000 / (5 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555), + 0x20000000000000 / (constant_55555 * 5), + 0x20000000000000 / (constant_55555 * 5 * 5), + 0x20000000000000 / (constant_55555 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * 5 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555), + 0x20000000000000 / (constant_55555 * constant_55555 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * 5 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * constant_55555), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5 * 5 * 5 * 5)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr double binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; + +#endif + +template struct binary_format_lookup_tables { + static constexpr float powers_of_ten[] = {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + + // Largest integer value v so that (5**index * v) <= 1<<24. + // 0x1000000 == 1<<24 + static constexpr uint64_t max_mantissa[] = { + 0x1000000, + 0x1000000 / 5, + 0x1000000 / (5 * 5), + 0x1000000 / (5 * 5 * 5), + 0x1000000 / (5 * 5 * 5 * 5), + 0x1000000 / (constant_55555), + 0x1000000 / (constant_55555 * 5), + 0x1000000 / (constant_55555 * 5 * 5), + 0x1000000 / (constant_55555 * 5 * 5 * 5), + 0x1000000 / (constant_55555 * 5 * 5 * 5 * 5), + 0x1000000 / (constant_55555 * constant_55555), + 0x1000000 / (constant_55555 * constant_55555 * 5)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr float binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; + +#endif + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + return 0; +#else + return -22; +#endif +} + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + return 0; +#else + return -10; +#endif +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 52; +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 23; +} + +template <> +inline constexpr int binary_format::max_exponent_round_to_even() { + return 23; +} + +template <> +inline constexpr int binary_format::max_exponent_round_to_even() { + return 10; +} + +template <> +inline constexpr int binary_format::min_exponent_round_to_even() { + return -4; +} + +template <> +inline constexpr int binary_format::min_exponent_round_to_even() { + return -17; +} + +template <> inline constexpr int binary_format::minimum_exponent() { + return -1023; +} + +template <> inline constexpr int binary_format::minimum_exponent() { + return -127; +} + +template <> inline constexpr int binary_format::infinite_power() { + return 0x7FF; +} + +template <> inline constexpr int binary_format::infinite_power() { + return 0xFF; +} + +template <> inline constexpr int binary_format::sign_index() { + return 63; +} + +template <> inline constexpr int binary_format::sign_index() { + return 31; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 22; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 10; +} + +template <> +inline constexpr uint64_t binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +template <> +inline constexpr uint64_t binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +// credit: Jakub Jelinek +#ifdef __STDCPP_FLOAT16_T__ +template struct binary_format_lookup_tables { + static constexpr std::float16_t powers_of_ten[] = {1e0f16, 1e1f16, 1e2f16, + 1e3f16, 1e4f16}; + + // Largest integer value v so that (5**index * v) <= 1<<11. + // 0x800 == 1<<11 + static constexpr uint64_t max_mantissa[] = {0x800, + 0x800 / 5, + 0x800 / (5 * 5), + 0x800 / (5 * 5 * 5), + 0x800 / (5 * 5 * 5 * 5), + 0x800 / (constant_55555)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr std::float16_t + binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t + binary_format_lookup_tables::max_mantissa[]; + +#endif + +template <> +inline constexpr std::float16_t +binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7C00; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x03FF; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x0400; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 4; +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 10; +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 4 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { + return 0; +} + +template <> +inline constexpr int +binary_format::max_exponent_round_to_even() { + return 5; +} + +template <> +inline constexpr int +binary_format::min_exponent_round_to_even() { + return -22; +} + +template <> +inline constexpr int binary_format::minimum_exponent() { + return -15; +} + +template <> +inline constexpr int binary_format::infinite_power() { + return 0x1F; +} + +template <> inline constexpr int binary_format::sign_index() { + return 15; +} + +template <> +inline constexpr int binary_format::largest_power_of_ten() { + return 4; +} + +template <> +inline constexpr int binary_format::smallest_power_of_ten() { + return -27; +} + +template <> +inline constexpr size_t binary_format::max_digits() { + return 22; +} +#endif // __STDCPP_FLOAT16_T__ + +// credit: Jakub Jelinek +#ifdef __STDCPP_BFLOAT16_T__ +template struct binary_format_lookup_tables { + static constexpr std::bfloat16_t powers_of_ten[] = {1e0bf16, 1e1bf16, 1e2bf16, + 1e3bf16}; + + // Largest integer value v so that (5**index * v) <= 1<<8. + // 0x100 == 1<<8 + static constexpr uint64_t max_mantissa[] = {0x100, 0x100 / 5, 0x100 / (5 * 5), + 0x100 / (5 * 5 * 5), + 0x100 / (5 * 5 * 5 * 5)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr std::bfloat16_t + binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t + binary_format_lookup_tables::max_mantissa[]; + +#endif + +template <> +inline constexpr std::bfloat16_t +binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 3; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7F80; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x007F; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x0080; +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 7; +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 3 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { + return 0; +} + +template <> +inline constexpr int +binary_format::max_exponent_round_to_even() { + return 3; +} + +template <> +inline constexpr int +binary_format::min_exponent_round_to_even() { + return -24; +} + +template <> +inline constexpr int binary_format::minimum_exponent() { + return -127; +} + +template <> +inline constexpr int binary_format::infinite_power() { + return 0xFF; +} + +template <> inline constexpr int binary_format::sign_index() { + return 15; +} + +template <> +inline constexpr int binary_format::largest_power_of_ten() { + return 38; +} + +template <> +inline constexpr int binary_format::smallest_power_of_ten() { + return -60; +} + +template <> +inline constexpr size_t binary_format::max_digits() { + return 98; +} +#endif // __STDCPP_BFLOAT16_T__ + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 22 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 10 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr double +binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> +inline constexpr float binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> inline constexpr int binary_format::largest_power_of_ten() { + return 308; +} + +template <> inline constexpr int binary_format::largest_power_of_ten() { + return 38; +} + +template <> +inline constexpr int binary_format::smallest_power_of_ten() { + return -342; +} + +template <> inline constexpr int binary_format::smallest_power_of_ten() { + return -64; +} + +template <> inline constexpr size_t binary_format::max_digits() { + return 769; +} + +template <> inline constexpr size_t binary_format::max_digits() { + return 114; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7F800000; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7FF0000000000000; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x007FFFFF; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x000FFFFFFFFFFFFF; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x00800000; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x0010000000000000; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +to_float(bool negative, adjusted_mantissa am, T &value) { + using equiv_uint = equiv_uint_t; + equiv_uint word = equiv_uint(am.mantissa); + word = equiv_uint(word | equiv_uint(am.power2) + << binary_format::mantissa_explicit_bits()); + word = + equiv_uint(word | equiv_uint(negative) << binary_format::sign_index()); +#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST + value = std::bit_cast(word); +#else + ::memcpy(&value, &word, sizeof(T)); +#endif +} + +template struct space_lut { + static constexpr bool value[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template constexpr bool space_lut::value[]; + +#endif + +template constexpr bool is_space(UC c) { + // wchar_t and char can be signed, so a negative code unit slips past a plain + // `c < 256` and then indexes the table by its truncated low byte. Compare as + // unsigned, matching the care taken in ch_to_digit. + using UnsignedUC = typename std::make_unsigned::type; + return static_cast(c) < 256 && space_lut<>::value[uint8_t(c)]; +} + +template static constexpr uint64_t int_cmp_zeros() { + static_assert((sizeof(UC) == 1) || (sizeof(UC) == 2) || (sizeof(UC) == 4), + "Unsupported character size"); + return (sizeof(UC) == 1) ? 0x3030303030303030 + : (sizeof(UC) == 2) + ? (uint64_t(UC('0')) << 48 | uint64_t(UC('0')) << 32 | + uint64_t(UC('0')) << 16 | UC('0')) + : (uint64_t(UC('0')) << 32 | UC('0')); +} + +template static constexpr int int_cmp_len() { + return sizeof(uint64_t) / sizeof(UC); +} + +template constexpr UC const *str_const_nan(); + +template <> constexpr char const *str_const_nan() { return "nan"; } + +template <> constexpr wchar_t const *str_const_nan() { return L"nan"; } + +template <> constexpr char16_t const *str_const_nan() { + return u"nan"; +} + +template <> constexpr char32_t const *str_const_nan() { + return U"nan"; +} + +#ifdef __cpp_char8_t +template <> constexpr char8_t const *str_const_nan() { + return u8"nan"; +} +#endif + +template constexpr UC const *str_const_inf(); + +template <> constexpr char const *str_const_inf() { return "infinity"; } + +template <> constexpr wchar_t const *str_const_inf() { + return L"infinity"; +} + +template <> constexpr char16_t const *str_const_inf() { + return u"infinity"; +} + +template <> constexpr char32_t const *str_const_inf() { + return U"infinity"; +} + +#ifdef __cpp_char8_t +template <> constexpr char8_t const *str_const_inf() { + return u8"infinity"; +} +#endif + +template struct int_luts { + static constexpr uint8_t chdigit[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + + static constexpr size_t maxdigits_u64[] = { + 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, 16, 16, 16, + 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 13, 13, 13, 13, 13, 13}; + + static constexpr uint64_t min_safe_u64[] = { + 9223372036854775808ull, 12157665459056928801ull, 4611686018427387904, + 7450580596923828125, 4738381338321616896, 3909821048582988049, + 9223372036854775808ull, 12157665459056928801ull, 10000000000000000000ull, + 5559917313492231481, 2218611106740436992, 8650415919381337933, + 2177953337809371136, 6568408355712890625, 1152921504606846976, + 2862423051509815793, 6746640616477458432, 15181127029874798299ull, + 1638400000000000000, 3243919932521508681, 6221821273427820544, + 11592836324538749809ull, 876488338465357824, 1490116119384765625, + 2481152873203736576, 4052555153018976267, 6502111422497947648, + 10260628712958602189ull, 15943230000000000000ull, 787662783788549761, + 1152921504606846976, 1667889514952984961, 2386420683693101056, + 3379220508056640625, 4738381338321616896}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template constexpr uint8_t int_luts::chdigit[]; + +template constexpr size_t int_luts::maxdigits_u64[]; + +template constexpr uint64_t int_luts::min_safe_u64[]; + +#endif + +template +simdjson_fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) { + // wchar_t and char can be signed, so we need to be careful. + using UnsignedUC = typename std::make_unsigned::type; + return int_luts<>::chdigit[static_cast( + static_cast(c) & + static_cast( + -((static_cast(c) & ~0xFFull) == 0)))]; +} + +simdjson_fastfloat_really_inline constexpr size_t max_digits_u64(int base) { + return int_luts<>::maxdigits_u64[base - 2]; +} + +// If a u64 is exactly max_digits_u64() in length, this is +// the value below which it has definitely overflowed. +simdjson_fastfloat_really_inline constexpr uint64_t min_safe_u64(int base) { + return int_luts<>::min_safe_u64[base - 2]; +} + +static_assert(std::is_same, uint64_t>::value, + "equiv_uint should be uint64_t for double"); +static_assert(std::numeric_limits::is_iec559, + "double must fulfill the requirements of IEC 559 (IEEE 754)"); + +static_assert(std::is_same, uint32_t>::value, + "equiv_uint should be uint32_t for float"); +static_assert(std::numeric_limits::is_iec559, + "float must fulfill the requirements of IEC 559 (IEEE 754)"); + +#ifdef __STDCPP_FLOAT64_T__ +static_assert(std::is_same, uint64_t>::value, + "equiv_uint should be uint64_t for std::float64_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::float64_t must fulfill the requirements of IEC 559 (IEEE 754)"); + +template <> +struct binary_format : public binary_format {}; +#endif // __STDCPP_FLOAT64_T__ + +#ifdef __STDCPP_FLOAT32_T__ +static_assert(std::is_same, uint32_t>::value, + "equiv_uint should be uint32_t for std::float32_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::float32_t must fulfill the requirements of IEC 559 (IEEE 754)"); + +template <> +struct binary_format : public binary_format {}; +#endif // __STDCPP_FLOAT32_T__ + +#ifdef __STDCPP_FLOAT16_T__ +static_assert( + std::is_same::equiv_uint, uint16_t>::value, + "equiv_uint should be uint16_t for std::float16_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::float16_t must fulfill the requirements of IEC 559 (IEEE 754)"); +#endif // __STDCPP_FLOAT16_T__ + +#ifdef __STDCPP_BFLOAT16_T__ +static_assert( + std::is_same::equiv_uint, uint16_t>::value, + "equiv_uint should be uint16_t for std::bfloat16_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::bfloat16_t must fulfill the requirements of IEC 559 (IEEE 754)"); +#endif // __STDCPP_BFLOAT16_T__ + +constexpr chars_format operator~(chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(~static_cast(rhs)); +} + +constexpr chars_format operator&(chars_format lhs, chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(static_cast(lhs) & + static_cast(rhs)); +} + +constexpr chars_format operator|(chars_format lhs, chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(static_cast(lhs) | + static_cast(rhs)); +} + +constexpr chars_format operator^(chars_format lhs, chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(static_cast(lhs) ^ + static_cast(rhs)); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & +operator&=(chars_format &lhs, chars_format rhs) noexcept { + return lhs = (lhs & rhs); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & +operator|=(chars_format &lhs, chars_format rhs) noexcept { + return lhs = (lhs | rhs); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & +operator^=(chars_format &lhs, chars_format rhs) noexcept { + return lhs = (lhs ^ rhs); +} + +namespace detail { +// adjust for deprecated feature macros +constexpr chars_format adjust_for_feature_macros(chars_format fmt) { + return fmt +#ifdef SIMDJSON_FASTFLOAT_ALLOWS_LEADING_PLUS + | chars_format::allow_leading_plus +#endif +#ifdef SIMDJSON_FASTFLOAT_SKIP_WHITE_SPACE + | chars_format::skip_white_space +#endif + ; +} +} // namespace detail +} // namespace simdjson_fast_float + +#endif + + +#ifndef SIMDJSON_FASTFLOAT_FAST_FLOAT_H +#define SIMDJSON_FASTFLOAT_FAST_FLOAT_H + + +namespace simdjson_fast_float { +/** + * This function parses the character sequence [first,last) for a number. It + * parses floating-point numbers expecting a locale-independent format + * equivalent to what is used by std::strtod in the default ("C") locale. The + * resulting floating-point value is the closest floating-point values (using + * either float or double), using the "round to even" convention for values that + * would otherwise fall right in-between two values. That is, we provide exact + * parsing according to the IEEE standard. + * + * Given a successful parse, the pointer (`ptr`) in the returned value is set to + * point right after the parsed number, and the `value` referenced is set to the + * parsed value. In case of error, the returned `ec` contains a representative + * error, otherwise the default (`std::errc()`) value is stored. + * + * The implementation does not throw and does not allocate memory (e.g., with + * `new` or `malloc`). + * + * Like the C++17 standard, the `simdjson_fast_float::from_chars` functions take an + * optional last argument of the type `simdjson_fast_float::chars_format`. It is a bitset + * value: we check whether `fmt & simdjson_fast_float::chars_format::fixed` and `fmt & + * simdjson_fast_float::chars_format::scientific` are set to determine whether we allow + * the fixed point and scientific notation respectively. The default is + * `simdjson_fast_float::chars_format::general` which allows both `fixed` and + * `scientific`. + */ +template ::value)> +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, + chars_format fmt = chars_format::general) noexcept; + +/** + * Like from_chars, but accepts an `options` argument to govern number parsing. + * Both for floating-point types and integer types. + */ +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept; + +/** + * This function multiplies an integer number by a power of 10 and returns + * the result as a double precision floating-point value that is correctly + * rounded. The resulting floating-point value is the closest floating-point + * value, using the "round to nearest, tie to even" convention for values that + * would otherwise fall right in-between two values. That is, we provide exact + * conversion according to the IEEE standard. + * + * On overflow infinity is returned, on underflow 0 is returned. + * + * The implementation does not throw and does not allocate memory (e.g., with + * `new` or `malloc`). + */ +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept; +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept; + +/** + * This function is a template overload of `integer_times_pow10()` + * that returns a floating-point value of type `T` that is one of + * supported floating-point types (e.g. `double`, `float`). + */ +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept; +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept; + +/** + * from_chars for integer types. + */ +template ::value)> +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, int base = 10) noexcept; + +} // namespace simdjson_fast_float + +#endif // SIMDJSON_FASTFLOAT_FAST_FLOAT_H + +#ifndef SIMDJSON_FASTFLOAT_ASCII_NUMBER_H +#define SIMDJSON_FASTFLOAT_ASCII_NUMBER_H + +#include +#include +#include +#include +#include +#include + + +#ifdef SIMDJSON_FASTFLOAT_SSE2 +#include +#endif + +#ifdef SIMDJSON_FASTFLOAT_NEON +#include +#endif + +namespace simdjson_fast_float { + +template simdjson_fastfloat_really_inline constexpr bool has_simd_opt() { +#ifdef SIMDJSON_FASTFLOAT_HAS_SIMD + return std::is_same::value; +#else + return false; +#endif +} + +// Next function can be micro-optimized, but compilers are entirely +// able to optimize it well. +template +simdjson_fastfloat_really_inline constexpr bool is_integer(UC c) noexcept { + return static_cast(c - UC('0')) <= 9u; +} + +simdjson_fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) { + return (val & 0xFF00000000000000) >> 56 | (val & 0x00FF000000000000) >> 40 | + (val & 0x0000FF0000000000) >> 24 | (val & 0x000000FF00000000) >> 8 | + (val & 0x00000000FF000000) << 8 | (val & 0x0000000000FF0000) << 24 | + (val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56; +} + +simdjson_fastfloat_really_inline constexpr uint32_t byteswap_32(uint32_t val) { + return (val >> 24) | ((val >> 8) & 0x0000FF00u) | ((val << 8) & 0x00FF0000u) | + (val << 24); +} + +// Read 8 UC into a u64. Truncates UC if not char. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +read8_to_u64(UC const *chars) { + if (cpp20_and_in_constexpr() || !std::is_same::value) { + uint64_t val = 0; + for (int i = 0; i < 8; ++i) { + val |= uint64_t(uint8_t(*chars)) << (i * 8); + ++chars; + } + return val; + } + uint64_t val; + ::memcpy(&val, chars, sizeof(uint64_t)); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN == 1 + // Need to read as-if the number was in little-endian order. + val = byteswap(val); +#endif + return val; +} + +// Read 4 UC into a u32. Truncates UC if not char. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint32_t +read4_to_u32(UC const *chars) { + if (cpp20_and_in_constexpr() || !std::is_same::value) { + uint32_t val = 0; + for (int i = 0; i < 4; ++i) { + val |= uint32_t(uint8_t(*chars)) << (i * 8); + ++chars; + } + return val; + } + uint32_t val; + ::memcpy(&val, chars, sizeof(uint32_t)); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN == 1 + val = byteswap_32(val); +#endif + return val; +} +#ifdef SIMDJSON_FASTFLOAT_SSE2 + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(__m128i const data) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + __m128i const packed = _mm_packus_epi16(data, data); +#ifdef SIMDJSON_FASTFLOAT_64BIT + return uint64_t(_mm_cvtsi128_si64(packed)); +#else + uint64_t value; + // Visual Studio + older versions of GCC don't support _mm_storeu_si64 + _mm_storel_epi64(reinterpret_cast<__m128i *>(&value), packed); + return value; +#endif + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + return simd_read8_to_u64( + _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars))); + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +#elif defined(SIMDJSON_FASTFLOAT_NEON) + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(uint16x8_t const data) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + uint8x8_t utf8_packed = vmovn_u16(data); + return vget_lane_u64(vreinterpret_u64_u8(utf8_packed), 0); + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + return simd_read8_to_u64( + vld1q_u16(reinterpret_cast(chars))); + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +#endif // SIMDJSON_FASTFLOAT_SSE2 + +// MSVC SFINAE is broken pre-VS2017 +#if defined(_MSC_VER) && _MSC_VER <= 1900 +template +#else +template ()) = 0> +#endif +// dummy for compile +uint64_t simd_read8_to_u64(UC const *) { + return 0; +} + +// credit @aqrit +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint32_t +parse_eight_digits_unrolled(uint64_t val) { + uint64_t const mask = 0x000000FF000000FF; + uint64_t const mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) + uint64_t const mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) + val -= 0x3030303030303030; + val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; + val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; + return uint32_t(val); +} + +// Call this if chars are definitely 8 digits. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint32_t +parse_eight_digits_unrolled(UC const *chars) noexcept { + if (cpp20_and_in_constexpr() || !has_simd_opt()) { + return parse_eight_digits_unrolled(read8_to_u64(chars)); // truncation okay + } + return parse_eight_digits_unrolled(simd_read8_to_u64(chars)); +} + +// credit @aqrit +simdjson_fastfloat_really_inline constexpr bool +is_made_of_eight_digits_fast(uint64_t val) noexcept { + return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & + 0x8080808080808080)); +} + +simdjson_fastfloat_really_inline constexpr bool +is_made_of_four_digits_fast(uint32_t val) noexcept { + return !((((val + 0x46464646) | (val - 0x30303030)) & 0x80808080)); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint32_t +parse_four_digits_unrolled(uint32_t val) noexcept { + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + +#ifdef SIMDJSON_FASTFLOAT_HAS_SIMD + +// Call this if chars might not be 8 digits. +// Using this style (instead of is_made_of_eight_digits_fast() then +// parse_eight_digits_unrolled()) ensures we don't load SIMD registers twice. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +simd_parse_if_eight_digits_unrolled(char16_t const *chars, + uint64_t &i) noexcept { + if (cpp20_and_in_constexpr()) { + return false; + } +#ifdef SIMDJSON_FASTFLOAT_SSE2 + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + __m128i const data = + _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars)); + + // (x - '0') <= 9 + // http://0x80.pl/articles/simd-parsing-int-sequences.html + __m128i const t0 = _mm_add_epi16(data, _mm_set1_epi16(32720)); + __m128i const t1 = _mm_cmpgt_epi16(t0, _mm_set1_epi16(-32759)); + + if (_mm_movemask_epi8(t1) == 0) { + i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); + return true; + } else + return false; + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +#elif defined(SIMDJSON_FASTFLOAT_NEON) + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + uint16x8_t const data = vld1q_u16(reinterpret_cast(chars)); + + // (x - '0') <= 9 + // http://0x80.pl/articles/simd-parsing-int-sequences.html + uint16x8_t const t0 = vsubq_u16(data, vmovq_n_u16('0')); + uint16x8_t const mask = vcltq_u16(t0, vmovq_n_u16('9' - '0' + 1)); + + if (vminvq_u16(mask) == 0xFFFF) { + i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); + return true; + } else + return false; + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +#else + static_cast(chars); + static_cast(i); + return false; +#endif // SIMDJSON_FASTFLOAT_SSE2 +} + +#endif // SIMDJSON_FASTFLOAT_HAS_SIMD + +// MSVC SFINAE is broken pre-VS2017 +#if defined(_MSC_VER) && _MSC_VER <= 1900 +template +#else +template ()) = 0> +#endif +// dummy for compile +bool simd_parse_if_eight_digits_unrolled(UC const *, uint64_t &) { + return 0; +} + +template ::value) = 0> +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +loop_parse_if_eight_digits(UC const *&p, UC const *const pend, uint64_t &i) { + if (!has_simd_opt()) { + return; + } + while ((std::distance(p, pend) >= 8) && + simd_parse_if_eight_digits_unrolled( + p, i)) { // in rare cases, this will overflow, but that's ok + p += 8; + } +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +loop_parse_if_eight_digits(char const *&p, char const *const pend, + uint64_t &i) { + // optimizes better than parse_if_eight_digits_unrolled() for UC = char. + while ((std::distance(p, pend) >= 8) && + is_made_of_eight_digits_fast(read8_to_u64(p))) { + i = i * 100000000 + + parse_eight_digits_unrolled(read8_to_u64( + p)); // in rare cases, this will overflow, but that's ok + p += 8; + } + // Consume a remaining 4-7 digit run in a single SWAR step instead of + // byte-by-byte (reuses the existing 4-digit helpers). The parsed result is + // identical either way. Historically gated to clang because gcc regressed on + // short remainders, but that verdict predates the span-elision restructure; + // with the leaner hot path the 4-digit step now wins on gcc as well. + if ((pend - p) >= 4) { + uint32_t const val4 = read4_to_u32(p); + if (is_made_of_four_digits_fast(val4)) { + i = i * 10000 + + parse_four_digits_unrolled(val4); // may overflow, that's ok + p += 4; + } + } +} + +enum class parse_error { + no_error, + // [JSON-only] The minus sign must be followed by an integer. + missing_integer_after_sign, + // A sign must be followed by an integer or dot. + missing_integer_or_dot_after_sign, + // [JSON-only] The integer part must not have leading zeros. + leading_zeros_in_integer_part, + // [JSON-only] The integer part must have at least one digit. + no_digits_in_integer_part, + // [JSON-only] If there is a decimal point, there must be digits in the + // fractional part. + no_digits_in_fractional_part, + // The mantissa must have at least one digit. + no_digits_in_mantissa, + // Scientific notation requires an exponential part. + missing_exponential_part, +}; + +template struct parsed_number_string_t { + int64_t exponent{0}; + uint64_t mantissa{0}; + UC const *lastmatch{nullptr}; + bool negative{false}; + bool valid{false}; + bool too_many_digits{false}; + // contains the range of the significant digits + span integer{}; // non-nullable + span fraction{}; // nullable + parse_error error{parse_error::no_error}; +}; + +using byte_span = span; +using parsed_number_string = parsed_number_string_t; + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 parsed_number_string_t +report_parse_error(UC const *p, parse_error error) { + parsed_number_string_t answer; + answer.valid = false; + answer.lastmatch = p; + answer.error = error; + return answer; +} + +// Assuming that you use no more than 19 digits, this will +// parse an ASCII string. +// +// store_spans is a *runtime* flag (not a template parameter, deliberately: a +// template would create a second instantiation of this whole function and the +// extra icache pressure wipes out the gain). When false, the integer/fraction +// spans (read only by the rare digit_comp slow path) are not materialized, +// which keeps the fat parsed_number_string_t off the hot path. The caller +// re-parses with store_spans=true if the slow path is actually reached. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 parsed_number_string_t +parse_number_string(UC const *p, UC const *pend, parse_options_t options, + bool store_spans = true) noexcept { + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + UC const decimal_point = options.decimal_point; + + parsed_number_string_t answer; + answer.valid = false; + answer.too_many_digits = false; + // assume p < pend, so dereference without checks; + answer.negative = (*p == UC('-')); + // C++17 20.19.3.(7.1) explicitly forbids '+' sign here + if ((*p == UC('-')) || (uint64_t(fmt & chars_format::allow_leading_plus) && + !basic_json_fmt && *p == UC('+'))) { + ++p; + if (p == pend) { + return report_parse_error( + p, parse_error::missing_integer_or_dot_after_sign); + } + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { + if (!is_integer(*p)) { // a sign must be followed by an integer + return report_parse_error(p, + parse_error::missing_integer_after_sign); + } + } + else { + if (!is_integer(*p) && + (*p != + decimal_point)) { // a sign must be followed by an integer or the dot + return report_parse_error( + p, parse_error::missing_integer_or_dot_after_sign); + } + } + } + UC const *const start_digits = p; + + uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad) + + // Straight-line unroll of the integer-part scan: most integer parts are + // 1-5 digits, so peeling the first iterations eliminates the loop back-edge + // for the common case. Semantics are identical to the original `while` loop: + // i = 10*i + digit, advancing p. + if ((p != pend) && is_integer(*p)) { + i = uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + while ((p != pend) && is_integer(*p)) { + // a multiplication by 10 is cheaper than an arbitrary integer + // multiplication + i = 10 * i + + uint64_t(*p - UC('0')); // might overflow, handled later + ++p; + } + } + } + } + } + } + UC const *const end_of_integer_part = p; + int64_t digit_count = int64_t(end_of_integer_part - start_digits); + if (store_spans) { + answer.integer = span(start_digits, size_t(digit_count)); + } + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { + // at least 1 digit in integer part, without leading zeros + if (digit_count == 0) { + return report_parse_error(p, parse_error::no_digits_in_integer_part); + } + if ((start_digits[0] == UC('0') && digit_count > 1)) { + return report_parse_error(start_digits, + parse_error::leading_zeros_in_integer_part); + } + } + + int64_t exponent = 0; + bool const has_decimal_point = (p != pend) && (*p == decimal_point); + if (has_decimal_point) { + ++p; + UC const *before = p; + // can occur at most twice without overflowing, but let it occur more, since + // for integers with many digits, digit parsing is the primary bottleneck. + loop_parse_if_eight_digits(p, pend, i); + + while ((p != pend) && is_integer(*p)) { + uint8_t digit = uint8_t(*p - UC('0')); + ++p; + i = i * 10 + digit; // in rare cases, this will overflow, but that's ok + } + exponent = before - p; + if (store_spans) { + answer.fraction = span(before, size_t(p - before)); + } + digit_count -= exponent; + } + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { + // at least 1 digit in fractional part + if (has_decimal_point && exponent == 0) { + return report_parse_error(p, + parse_error::no_digits_in_fractional_part); + } + } + else if (digit_count == 0) { // we must have encountered at least one integer! + return report_parse_error(p, parse_error::no_digits_in_mantissa); + } + int64_t exp_number = 0; // explicit exponential part + if ((uint64_t(fmt & chars_format::scientific) && (p != pend) && + ((UC('e') == *p) || (UC('E') == *p))) || + (uint64_t(fmt & detail::basic_fortran_fmt) && (p != pend) && + ((UC('+') == *p) || (UC('-') == *p) || (UC('d') == *p) || + (UC('D') == *p)))) { + UC const *location_of_e = p; + if ((UC('e') == *p) || (UC('E') == *p) || (UC('d') == *p) || + (UC('D') == *p)) { + ++p; + } + bool neg_exp = false; + if ((p != pend) && (UC('-') == *p)) { + neg_exp = true; + ++p; + } else if ((p != pend) && + (UC('+') == + *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1) + ++p; + } + if ((p == pend) || !is_integer(*p)) { + if (!uint64_t(fmt & chars_format::fixed)) { + // The exponential part is invalid for scientific notation, so it must + // be a trailing token for fixed notation. However, fixed notation is + // disabled, so report a scientific notation error. + return report_parse_error(p, parse_error::missing_exponential_part); + } + // Otherwise, we will be ignoring the 'e'. + p = location_of_e; + } else { + while ((p != pend) && is_integer(*p)) { + uint8_t digit = uint8_t(*p - UC('0')); + if (exp_number < 0x10000000) { + exp_number = 10 * exp_number + digit; + } + ++p; + } + if (neg_exp) { + exp_number = -exp_number; + } + exponent += exp_number; + } + } else { + // If it scientific and not fixed, we have to bail out. + if (uint64_t(fmt & chars_format::scientific) && + !uint64_t(fmt & chars_format::fixed)) { + return report_parse_error(p, parse_error::missing_exponential_part); + } + } + answer.lastmatch = p; + answer.valid = true; + + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon. + // + // We can deal with up to 19 digits. + if (digit_count > 19) { // this is uncommon + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + // We need to be mindful of the case where we only have zeroes... + // E.g., 0.000000000...000. + UC const *start = start_digits; + while ((start != pend) && (*start == UC('0') || *start == decimal_point)) { + if (*start == UC('0')) { + digit_count--; + } + start++; + } + + if (digit_count > 19) { + answer.too_many_digits = true; + // The truncation recompute below reads the integer/fraction spans. When + // store_spans is false we didn't materialize them, so just flag + // too_many_digits; the caller re-parses with store_spans=true to obtain + // the corrected mantissa/exponent before taking the slow path. + if (store_spans) { + // Let us start again, this time, avoiding overflows. + // We don't need to call if is_integer, since we use the + // pre-tokenized spans from above. + i = 0; + p = answer.integer.ptr; + UC const *int_end = p + answer.integer.len(); + uint64_t const minimal_nineteen_digit_integer{1000000000000000000}; + while ((i < minimal_nineteen_digit_integer) && (p != int_end)) { + i = i * 10 + uint64_t(*p - UC('0')); + ++p; + } + if (i >= minimal_nineteen_digit_integer) { // We have a big integer + exponent = end_of_integer_part - p + exp_number; + } else { // We have a value with a fractional component. + p = answer.fraction.ptr; + UC const *frac_end = p + answer.fraction.len(); + while ((i < minimal_nineteen_digit_integer) && (p != frac_end)) { + i = i * 10 + uint64_t(*p - UC('0')); + ++p; + } + exponent = answer.fraction.ptr - p + exp_number; + } + // We have now corrected both exponent and i, to a truncated value + } + } + } + answer.exponent = exponent; + answer.mantissa = i; + return answer; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +parse_int_string(UC const *p, UC const *pend, T &value, + parse_options_t options) { + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + int const base = options.base; + + from_chars_result_t answer; + + UC const *const first = p; + + bool const negative = (*p == UC('-')); +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(push) +#pragma warning(disable : 4127) +#endif + if (!std::is_signed::value && negative) { +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(pop) +#endif + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + if ((*p == UC('-')) || + (uint64_t(fmt & chars_format::allow_leading_plus) && (*p == UC('+')))) { + ++p; + } + + UC const *const start_num = p; + + while (p != pend && *p == UC('0')) { + ++p; + } + + bool const has_leading_zeros = p > start_num; + + UC const *const start_digits = p; + + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17( + (std::is_same::value && sizeof(UC) == 1)) { + if (base == 10) { + const size_t len = static_cast(pend - p); + if (len == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + } else { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + } + return answer; + } + + uint32_t digits; + +#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && SIMDJSON_FASTFLOAT_HAS_BIT_CAST + if (std::is_constant_evaluated()) { + uint8_t str[4]{}; + for (size_t j = 0; j < 4 && j < len; ++j) { + str[j] = static_cast(p[j]); + } + digits = std::bit_cast(str); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN + digits = byteswap_32(digits); +#endif + } +#else + if (false) { + } +#endif + else if (len >= 4) { + ::memcpy(&digits, p, 4); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN + digits = byteswap_32(digits); +#endif + } else { + uint32_t b0 = static_cast(p[0]); + uint32_t b1 = (len > 1) ? static_cast(p[1]) : 0xFFu; + uint32_t b2 = (len > 2) ? static_cast(p[2]) : 0xFFu; + uint32_t b3 = 0xFFu; + digits = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24); + } + + uint32_t magic = + ((digits + 0x46464646u) | (digits - 0x30303030u)) & 0x80808080u; + uint32_t tz = + static_cast(countr_zero_32(magic)); // 7, 15, 23, 31, or 32 + uint32_t nd = (tz == 32) ? 4 : (tz >> 3); + nd = static_cast(nd < len ? nd : len); + if (nd == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + return answer; + } + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + if (nd > 3) { + const UC *q = p + nd; + size_t rem = len - nd; + while (rem) { + if (*q < UC('0') || *q > UC('9')) + break; + ++q; + --rem; + } + answer.ec = std::errc::result_out_of_range; + answer.ptr = q; + return answer; + } + + digits ^= 0x30303030u; + digits <<= ((4 - nd) * 8); + + uint32_t check = ((digits >> 24) & 0xff) | ((digits >> 8) & 0xff00) | + ((digits << 8) & 0xff0000); + if (check > 0x00020505) { + answer.ec = std::errc::result_out_of_range; + answer.ptr = p + nd; + return answer; + } + value = static_cast((0x640a01 * digits) >> 24); + answer.ec = std::errc(); + answer.ptr = p + nd; + return answer; + } + } + + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17( + (std::is_same::value && sizeof(UC) == 1)) { + if (base == 10) { + const size_t len = size_t(pend - p); + if (len == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + } else { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + } + return answer; + } + + if (len >= 4) { + uint32_t digits = read4_to_u32(p); + if (is_made_of_four_digits_fast(digits)) { + uint32_t v = parse_four_digits_unrolled(digits); + if (len >= 5 && is_integer(p[4])) { + v = v * 10 + uint32_t(p[4] - '0'); + if (len >= 6 && is_integer(p[5])) { + answer.ec = std::errc::result_out_of_range; + const UC *q = p + 5; + while (q != pend && is_integer(*q)) { + q++; + } + answer.ptr = q; + return answer; + } + if (v > 65535) { + answer.ec = std::errc::result_out_of_range; + answer.ptr = p + 5; + return answer; + } + value = uint16_t(v); + answer.ec = std::errc(); + answer.ptr = p + 5; + return answer; + } + // 4 digits + value = uint16_t(v); + answer.ec = std::errc(); + answer.ptr = p + 4; + return answer; + } + } + } + } + + uint64_t i = 0; + if (base == 10) { + loop_parse_if_eight_digits(p, pend, i); // use SIMD if possible + } + while (p != pend) { + uint8_t digit = ch_to_digit(*p); + if (digit >= base) { + break; + } + i = uint64_t(base) * i + digit; // might overflow, check this later + p++; + } + + size_t digit_count = size_t(p - start_digits); + + if (digit_count == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + } else { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + } + return answer; + } + + answer.ptr = p; + + // check u64 overflow + size_t max_digits = max_digits_u64(base); + if (digit_count > max_digits) { + answer.ec = std::errc::result_out_of_range; + return answer; + } + // this check can be eliminated for all other types, but they will all require + // a max_digits(base) equivalent + if (digit_count == max_digits) { + // At the max_digits boundary the accumulator `i` may have wrapped around + // 2^64. A plain `i < min_safe_u64(base)` test is not sufficient: for any + // base whose max_digits-length range exceeds 2^64 (base 10 reaches + // ~5.4 * 2^64 at 20 digits) the value can wrap a whole multiple of 2^64 and + // land back above min_safe, slipping through. Decide exactly in O(1) using + // the leading digit, following the approach used in simdjson: + // ms == min_safe_u64(base) == base^(max_digits-1), the smallest + // max_digits-length value. + // dmax == the largest leading digit whose number can still fit in u64. + // The leading-digit band [d*ms, (d+1)*ms) has width ms < 2^64, so within + // the single band where d == dmax the value straddles 2^64 at most once, + // and a single threshold separates wrapped from non-wrapped values. A + // leading digit above dmax always overflows; below dmax always fits. + uint64_t const ms = min_safe_u64(base); + uint64_t const dmax = (std::numeric_limits::max)() / ms; + uint64_t const lead = ch_to_digit(*start_digits); + if (lead > dmax || (lead == dmax && i < dmax * ms)) { + answer.ec = std::errc::result_out_of_range; + return answer; + } + } + + // check other types overflow + if (!std::is_same::value) { + if (i > uint64_t((std::numeric_limits::max)()) + uint64_t(negative)) { + answer.ec = std::errc::result_out_of_range; + return answer; + } + } + + if (negative) { +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(push) +#pragma warning(disable : 4146) +#endif + // this weird workaround is required because: + // - converting unsigned to signed when its value is greater than signed max + // is UB pre-C++23. + // - reinterpret_casting (~i + 1) would work, but it is not constexpr + // this is always optimized into a neg instruction (note: T is an integer + // type) + value = T(-(std::numeric_limits::max)() - + T(i - uint64_t((std::numeric_limits::max)()))); +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(pop) +#endif + } else { + value = T(i); + } + + answer.ec = std::errc(); + return answer; +} + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_FAST_TABLE_H +#define SIMDJSON_FASTFLOAT_FAST_TABLE_H + +#include + +namespace simdjson_fast_float { + +/** + * When mapping numbers from decimal to binary, + * we go from w * 10^q to m * 2^p but we have + * 10^q = 5^q * 2^q, so effectively + * we are trying to match + * w * 2^q * 5^q to m * 2^p. Thus the powers of two + * are not a concern since they can be represented + * exactly using the binary notation, only the powers of five + * affect the binary significand. + */ + +/** + * The smallest non-zero float (binary64) is 2^-1074. + * We take as input numbers of the form w x 10^q where w < 2^64. + * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. + * However, we have that + * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^-1074. + * Thus it is possible for a number of the form w * 10^-342 where + * w is a 64-bit value to be a non-zero floating-point number. + ********* + * Any number of form w * 10^309 where w>= 1 is going to be + * infinite in binary64 so we never need to worry about powers + * of 5 greater than 308. + */ +template struct powers_template { + + constexpr static int smallest_power_of_five = + binary_format::smallest_power_of_ten(); + constexpr static int largest_power_of_five = + binary_format::largest_power_of_ten(); + constexpr static int number_of_entries = + 2 * (largest_power_of_five - smallest_power_of_five + 1); + // Powers of five from 5^-342 all the way to 5^308 rounded toward one. + constexpr static uint64_t power_of_five_128[number_of_entries] = { + 0xeef453d6923bd65a, 0x113faa2906a13b3f, + 0x9558b4661b6565f8, 0x4ac7ca59a424c507, + 0xbaaee17fa23ebf76, 0x5d79bcf00d2df649, + 0xe95a99df8ace6f53, 0xf4d82c2c107973dc, + 0x91d8a02bb6c10594, 0x79071b9b8a4be869, + 0xb64ec836a47146f9, 0x9748e2826cdee284, + 0xe3e27a444d8d98b7, 0xfd1b1b2308169b25, + 0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7, + 0xb208ef855c969f4f, 0xbdbd2d335e51a935, + 0xde8b2b66b3bc4723, 0xad2c788035e61382, + 0x8b16fb203055ac76, 0x4c3bcb5021afcc31, + 0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d, + 0xd953e8624b85dd78, 0xd71d6dad34a2af0d, + 0x87d4713d6f33aa6b, 0x8672648c40e5ad68, + 0xa9c98d8ccb009506, 0x680efdaf511f18c2, + 0xd43bf0effdc0ba48, 0x212bd1b2566def2, + 0x84a57695fe98746d, 0x14bb630f7604b57, + 0xa5ced43b7e3e9188, 0x419ea3bd35385e2d, + 0xcf42894a5dce35ea, 0x52064cac828675b9, + 0x818995ce7aa0e1b2, 0x7343efebd1940993, + 0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8, + 0xca66fa129f9b60a6, 0xd41a26e077774ef6, + 0xfd00b897478238d0, 0x8920b098955522b4, + 0x9e20735e8cb16382, 0x55b46e5f5d5535b0, + 0xc5a890362fddbc62, 0xeb2189f734aa831d, + 0xf712b443bbd52b7b, 0xa5e9ec7501d523e4, + 0x9a6bb0aa55653b2d, 0x47b233c92125366e, + 0xc1069cd4eabe89f8, 0x999ec0bb696e840a, + 0xf148440a256e2c76, 0xc00670ea43ca250d, + 0x96cd2a865764dbca, 0x380406926a5e5728, + 0xbc807527ed3e12bc, 0xc605083704f5ecf2, + 0xeba09271e88d976b, 0xf7864a44c633682e, + 0x93445b8731587ea3, 0x7ab3ee6afbe0211d, + 0xb8157268fdae9e4c, 0x5960ea05bad82964, + 0xe61acf033d1a45df, 0x6fb92487298e33bd, + 0x8fd0c16206306bab, 0xa5d3b6d479f8e056, + 0xb3c4f1ba87bc8696, 0x8f48a4899877186c, + 0xe0b62e2929aba83c, 0x331acdabfe94de87, + 0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14, + 0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9, + 0xdb71e91432b1a24a, 0xc9e82cd9f69d6150, + 0x892731ac9faf056e, 0xbe311c083a225cd2, + 0xab70fe17c79ac6ca, 0x6dbd630a48aaf406, + 0xd64d3d9db981787d, 0x92cbbccdad5b108, + 0x85f0468293f0eb4e, 0x25bbf56008c58ea5, + 0xa76c582338ed2621, 0xaf2af2b80af6f24e, + 0xd1476e2c07286faa, 0x1af5af660db4aee1, + 0x82cca4db847945ca, 0x50d98d9fc890ed4d, + 0xa37fce126597973c, 0xe50ff107bab528a0, + 0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8, + 0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a, + 0x9faacf3df73609b1, 0x77b191618c54e9ac, + 0xc795830d75038c1d, 0xd59df5b9ef6a2417, + 0xf97ae3d0d2446f25, 0x4b0573286b44ad1d, + 0x9becce62836ac577, 0x4ee367f9430aec32, + 0xc2e801fb244576d5, 0x229c41f793cda73f, + 0xf3a20279ed56d48a, 0x6b43527578c1110f, + 0x9845418c345644d6, 0x830a13896b78aaa9, + 0xbe5691ef416bd60c, 0x23cc986bc656d553, + 0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8, + 0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9, + 0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53, + 0xe858ad248f5c22c9, 0xd1b3400f8f9cff68, + 0x91376c36d99995be, 0x23100809b9c21fa1, + 0xb58547448ffffb2d, 0xabd40a0c2832a78a, + 0xe2e69915b3fff9f9, 0x16c90c8f323f516c, + 0x8dd01fad907ffc3b, 0xae3da7d97f6792e3, + 0xb1442798f49ffb4a, 0x99cd11cfdf41779c, + 0xdd95317f31c7fa1d, 0x40405643d711d583, + 0x8a7d3eef7f1cfc52, 0x482835ea666b2572, + 0xad1c8eab5ee43b66, 0xda3243650005eecf, + 0xd863b256369d4a40, 0x90bed43e40076a82, + 0x873e4f75e2224e68, 0x5a7744a6e804a291, + 0xa90de3535aaae202, 0x711515d0a205cb36, + 0xd3515c2831559a83, 0xd5a5b44ca873e03, + 0x8412d9991ed58091, 0xe858790afe9486c2, + 0xa5178fff668ae0b6, 0x626e974dbe39a872, + 0xce5d73ff402d98e3, 0xfb0a3d212dc8128f, + 0x80fa687f881c7f8e, 0x7ce66634bc9d0b99, + 0xa139029f6a239f72, 0x1c1fffc1ebc44e80, + 0xc987434744ac874e, 0xa327ffb266b56220, + 0xfbe9141915d7a922, 0x4bf1ff9f0062baa8, + 0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9, + 0xc4ce17b399107c22, 0xcb550fb4384d21d3, + 0xf6019da07f549b2b, 0x7e2a53a146606a48, + 0x99c102844f94e0fb, 0x2eda7444cbfc426d, + 0xc0314325637a1939, 0xfa911155fefb5308, + 0xf03d93eebc589f88, 0x793555ab7eba27ca, + 0x96267c7535b763b5, 0x4bc1558b2f3458de, + 0xbbb01b9283253ca2, 0x9eb1aaedfb016f16, + 0xea9c227723ee8bcb, 0x465e15a979c1cadc, + 0x92a1958a7675175f, 0xbfacd89ec191ec9, + 0xb749faed14125d36, 0xcef980ec671f667b, + 0xe51c79a85916f484, 0x82b7e12780e7401a, + 0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810, + 0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15, + 0xdfbdcece67006ac9, 0x67a791e093e1d49a, + 0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0, + 0xaecc49914078536d, 0x58fae9f773886e18, + 0xda7f5bf590966848, 0xaf39a475506a899e, + 0x888f99797a5e012d, 0x6d8406c952429603, + 0xaab37fd7d8f58178, 0xc8e5087ba6d33b83, + 0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64, + 0x855c3be0a17fcd26, 0x5cf2eea09a55067f, + 0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e, + 0xd0601d8efc57b08b, 0xf13b94daf124da26, + 0x823c12795db6ce57, 0x76c53d08d6b70858, + 0xa2cb1717b52481ed, 0x54768c4b0c64ca6e, + 0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09, + 0xfe5d54150b090b02, 0xd3f93b35435d7c4c, + 0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf, + 0xc6b8e9b0709f109a, 0x359ab6419ca1091b, + 0xf867241c8cc6d4c0, 0xc30163d203c94b62, + 0x9b407691d7fc44f8, 0x79e0de63425dcf1d, + 0xc21094364dfb5636, 0x985915fc12f542e4, + 0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d, + 0x979cf3ca6cec5b5a, 0xa705992ceecf9c42, + 0xbd8430bd08277231, 0x50c6ff782a838353, + 0xece53cec4a314ebd, 0xa4f8bf5635246428, + 0x940f4613ae5ed136, 0x871b7795e136be99, + 0xb913179899f68584, 0x28e2557b59846e3f, + 0xe757dd7ec07426e5, 0x331aeada2fe589cf, + 0x9096ea6f3848984f, 0x3ff0d2c85def7621, + 0xb4bca50b065abe63, 0xfed077a756b53a9, + 0xe1ebce4dc7f16dfb, 0xd3e8495912c62894, + 0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c, + 0xb080392cc4349dec, 0xbd8d794d96aacfb3, + 0xdca04777f541c567, 0xecf0d7a0fc5583a0, + 0x89e42caaf9491b60, 0xf41686c49db57244, + 0xac5d37d5b79b6239, 0x311c2875c522ced5, + 0xd77485cb25823ac7, 0x7d633293366b828b, + 0x86a8d39ef77164bc, 0xae5dff9c02033197, + 0xa8530886b54dbdeb, 0xd9f57f830283fdfc, + 0xd267caa862a12d66, 0xd072df63c324fd7b, + 0x8380dea93da4bc60, 0x4247cb9e59f71e6d, + 0xa46116538d0deb78, 0x52d9be85f074e608, + 0xcd795be870516656, 0x67902e276c921f8b, + 0x806bd9714632dff6, 0xba1cd8a3db53b6, + 0xa086cfcd97bf97f3, 0x80e8a40eccd228a4, + 0xc8a883c0fdaf7df0, 0x6122cd128006b2cd, + 0xfad2a4b13d1b5d6c, 0x796b805720085f81, + 0x9cc3a6eec6311a63, 0xcbe3303674053bb0, + 0xc3f490aa77bd60fc, 0xbedbfc4411068a9c, + 0xf4f1b4d515acb93b, 0xee92fb5515482d44, + 0x991711052d8bf3c5, 0x751bdd152d4d1c4a, + 0xbf5cd54678eef0b6, 0xd262d45a78a0635d, + 0xef340a98172aace4, 0x86fb897116c87c34, + 0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0, + 0xbae0a846d2195712, 0x8974836059cca109, + 0xe998d258869facd7, 0x2bd1a438703fc94b, + 0x91ff83775423cc06, 0x7b6306a34627ddcf, + 0xb67f6455292cbf08, 0x1a3bc84c17b1d542, + 0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93, + 0x8e938662882af53e, 0x547eb47b7282ee9c, + 0xb23867fb2a35b28d, 0xe99e619a4f23aa43, + 0xdec681f9f4c31f31, 0x6405fa00e2ec94d4, + 0x8b3c113c38f9f37e, 0xde83bc408dd3dd04, + 0xae0b158b4738705e, 0x9624ab50b148d445, + 0xd98ddaee19068c76, 0x3badd624dd9b0957, + 0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6, + 0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c, + 0xd47487cc8470652b, 0x7647c3200069671f, + 0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073, + 0xa5fb0a17c777cf09, 0xf468107100525890, + 0xcf79cc9db955c2cc, 0x7182148d4066eeb4, + 0x81ac1fe293d599bf, 0xc6f14cd848405530, + 0xa21727db38cb002f, 0xb8ada00e5a506a7c, + 0xca9cf1d206fdc03b, 0xa6d90811f0e4851c, + 0xfd442e4688bd304a, 0x908f4a166d1da663, + 0x9e4a9cec15763e2e, 0x9a598e4e043287fe, + 0xc5dd44271ad3cdba, 0x40eff1e1853f29fd, + 0xf7549530e188c128, 0xd12bee59e68ef47c, + 0x9a94dd3e8cf578b9, 0x82bb74f8301958ce, + 0xc13a148e3032d6e7, 0xe36a52363c1faf01, + 0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1, + 0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9, + 0xbcb2b812db11a5de, 0x7415d448f6b6f0e7, + 0xebdf661791d60f56, 0x111b495b3464ad21, + 0x936b9fcebb25c995, 0xcab10dd900beec34, + 0xb84687c269ef3bfb, 0x3d5d514f40eea742, + 0xe65829b3046b0afa, 0xcb4a5a3112a5112, + 0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab, + 0xb3f4e093db73a093, 0x59ed216765690f56, + 0xe0f218b8d25088b8, 0x306869c13ec3532c, + 0x8c974f7383725573, 0x1e414218c73a13fb, + 0xafbd2350644eeacf, 0xe5d1929ef90898fa, + 0xdbac6c247d62a583, 0xdf45f746b74abf39, + 0x894bc396ce5da772, 0x6b8bba8c328eb783, + 0xab9eb47c81f5114f, 0x66ea92f3f326564, + 0xd686619ba27255a2, 0xc80a537b0efefebd, + 0x8613fd0145877585, 0xbd06742ce95f5f36, + 0xa798fc4196e952e7, 0x2c48113823b73704, + 0xd17f3b51fca3a7a0, 0xf75a15862ca504c5, + 0x82ef85133de648c4, 0x9a984d73dbe722fb, + 0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba, + 0xcc963fee10b7d1b3, 0x318df905079926a8, + 0xffbbcfe994e5c61f, 0xfdf17746497f7052, + 0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633, + 0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0, + 0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0, + 0x9c1661a651213e2d, 0x6bea10ca65c084e, + 0xc31bfa0fe5698db8, 0x486e494fcff30a62, + 0xf3e2f893dec3f126, 0x5a89dba3c3efccfa, + 0x986ddb5c6b3a76b7, 0xf89629465a75e01c, + 0xbe89523386091465, 0xf6bbb397f1135823, + 0xee2ba6c0678b597f, 0x746aa07ded582e2c, + 0x94db483840b717ef, 0xa8c2a44eb4571cdc, + 0xba121a4650e4ddeb, 0x92f34d62616ce413, + 0xe896a0d7e51e1566, 0x77b020baf9c81d17, + 0x915e2486ef32cd60, 0xace1474dc1d122e, + 0xb5b5ada8aaff80b8, 0xd819992132456ba, + 0xe3231912d5bf60e6, 0x10e1fff697ed6c69, + 0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1, + 0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2, + 0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde, + 0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b, + 0xad4ab7112eb3929d, 0x86c16c98d2c953c6, + 0xd89d64d57a607744, 0xe871c7bf077ba8b7, + 0x87625f056c7c4a8b, 0x11471cd764ad4972, + 0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf, + 0xd389b47879823479, 0x4aff1d108d4ec2c3, + 0x843610cb4bf160cb, 0xcedf722a585139ba, + 0xa54394fe1eedb8fe, 0xc2974eb4ee658828, + 0xce947a3da6a9273e, 0x733d226229feea32, + 0x811ccc668829b887, 0x806357d5a3f525f, + 0xa163ff802a3426a8, 0xca07c2dcb0cf26f7, + 0xc9bcff6034c13052, 0xfc89b393dd02f0b5, + 0xfc2c3f3841f17c67, 0xbbac2078d443ace2, + 0x9d9ba7832936edc0, 0xd54b944b84aa4c0d, + 0xc5029163f384a931, 0xa9e795e65d4df11, + 0xf64335bcf065d37d, 0x4d4617b5ff4a16d5, + 0x99ea0196163fa42e, 0x504bced1bf8e4e45, + 0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6, + 0xf07da27a82c37088, 0x5d767327bb4e5a4c, + 0x964e858c91ba2655, 0x3a6a07f8d510f86f, + 0xbbe226efb628afea, 0x890489f70a55368b, + 0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e, + 0x92c8ae6b464fc96f, 0x3b0b8bc90012929d, + 0xb77ada0617e3bbcb, 0x9ce6ebb40173744, + 0xe55990879ddcaabd, 0xcc420a6a101d0515, + 0x8f57fa54c2a9eab6, 0x9fa946824a12232d, + 0xb32df8e9f3546564, 0x47939822dc96abf9, + 0xdff9772470297ebd, 0x59787e2b93bc56f7, + 0x8bfbea76c619ef36, 0x57eb4edb3c55b65a, + 0xaefae51477a06b03, 0xede622920b6b23f1, + 0xdab99e59958885c4, 0xe95fab368e45eced, + 0x88b402f7fd75539b, 0x11dbcb0218ebb414, + 0xaae103b5fcd2a881, 0xd652bdc29f26a119, + 0xd59944a37c0752a2, 0x4be76d3346f0495f, + 0x857fcae62d8493a5, 0x6f70a4400c562ddb, + 0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952, + 0xd097ad07a71f26b2, 0x7e2000a41346a7a7, + 0x825ecc24c873782f, 0x8ed400668c0c28c8, + 0xa2f67f2dfa90563b, 0x728900802f0f32fa, + 0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9, + 0xfea126b7d78186bc, 0xe2f610c84987bfa8, + 0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9, + 0xc6ede63fa05d3143, 0x91503d1c79720dbb, + 0xf8a95fcf88747d94, 0x75a44c6397ce912a, + 0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba, + 0xc24452da229b021b, 0xfbe85badce996168, + 0xf2d56790ab41c2a2, 0xfae27299423fb9c3, + 0x97c560ba6b0919a5, 0xdccd879fc967d41a, + 0xbdb6b8e905cb600f, 0x5400e987bbc1c920, + 0xed246723473e3813, 0x290123e9aab23b68, + 0x9436c0760c86e30b, 0xf9a0b6720aaf6521, + 0xb94470938fa89bce, 0xf808e40e8d5b3e69, + 0xe7958cb87392c2c2, 0xb60b1d1230b20e04, + 0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2, + 0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3, + 0xe2280b6c20dd5232, 0x25c6da63c38de1b0, + 0x8d590723948a535f, 0x579c487e5a38ad0e, + 0xb0af48ec79ace837, 0x2d835a9df0c6d851, + 0xdcdb1b2798182244, 0xf8e431456cf88e65, + 0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff, + 0xac8b2d36eed2dac5, 0xe272467e3d222f3f, + 0xd7adf884aa879177, 0x5b0ed81dcc6abb0f, + 0x86ccbb52ea94baea, 0x98e947129fc2b4e9, + 0xa87fea27a539e9a5, 0x3f2398d747b36224, + 0xd29fe4b18e88640e, 0x8eec7f0d19a03aad, + 0x83a3eeeef9153e89, 0x1953cf68300424ac, + 0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7, + 0xcdb02555653131b6, 0x3792f412cb06794d, + 0x808e17555f3ebf11, 0xe2bbd88bbee40bd0, + 0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4, + 0xc8de047564d20a8b, 0xf245825a5a445275, + 0xfb158592be068d2e, 0xeed6e2f0f0d56712, + 0x9ced737bb6c4183d, 0x55464dd69685606b, + 0xc428d05aa4751e4c, 0xaa97e14c3c26b886, + 0xf53304714d9265df, 0xd53dd99f4b3066a8, + 0x993fe2c6d07b7fab, 0xe546a8038efe4029, + 0xbf8fdb78849a5f96, 0xde98520472bdd033, + 0xef73d256a5c0f77c, 0x963e66858f6d4440, + 0x95a8637627989aad, 0xdde7001379a44aa8, + 0xbb127c53b17ec159, 0x5560c018580d5d52, + 0xe9d71b689dde71af, 0xaab8f01e6e10b4a6, + 0x9226712162ab070d, 0xcab3961304ca70e8, + 0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22, + 0xe45c10c42a2b3b05, 0x8cb89a7db77c506a, + 0x8eb98a7a9a5b04e3, 0x77f3608e92adb242, + 0xb267ed1940f1c61c, 0x55f038b237591ed3, + 0xdf01e85f912e37a3, 0x6b6c46dec52f6688, + 0x8b61313bbabce2c6, 0x2323ac4b3b3da015, + 0xae397d8aa96c1b77, 0xabec975e0a0d081a, + 0xd9c7dced53c72255, 0x96e7bd358c904a21, + 0x881cea14545c7575, 0x7e50d64177da2e54, + 0xaa242499697392d2, 0xdde50bd1d5d0b9e9, + 0xd4ad2dbfc3d07787, 0x955e4ec64b44e864, + 0x84ec3c97da624ab4, 0xbd5af13bef0b113e, + 0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e, + 0xcfb11ead453994ba, 0x67de18eda5814af2, + 0x81ceb32c4b43fcf4, 0x80eacf948770ced7, + 0xa2425ff75e14fc31, 0xa1258379a94d028d, + 0xcad2f7f5359a3b3e, 0x96ee45813a04330, + 0xfd87b5f28300ca0d, 0x8bca9d6e188853fc, + 0x9e74d1b791e07e48, 0x775ea264cf55347e, + 0xc612062576589dda, 0x95364afe032a819e, + 0xf79687aed3eec551, 0x3a83ddbd83f52205, + 0x9abe14cd44753b52, 0xc4926a9672793543, + 0xc16d9a0095928a27, 0x75b7053c0f178294, + 0xf1c90080baf72cb1, 0x5324c68b12dd6339, + 0x971da05074da7bee, 0xd3f6fc16ebca5e04, + 0xbce5086492111aea, 0x88f4bb1ca6bcf585, + 0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6, + 0x9392ee8e921d5d07, 0x3aff322e62439fd0, + 0xb877aa3236a4b449, 0x9befeb9fad487c3, + 0xe69594bec44de15b, 0x4c2ebe687989a9b4, + 0x901d7cf73ab0acd9, 0xf9d37014bf60a11, + 0xb424dc35095cd80f, 0x538484c19ef38c95, + 0xe12e13424bb40e13, 0x2865a5f206b06fba, + 0x8cbccc096f5088cb, 0xf93f87b7442e45d4, + 0xafebff0bcb24aafe, 0xf78f69a51539d749, + 0xdbe6fecebdedd5be, 0xb573440e5a884d1c, + 0x89705f4136b4a597, 0x31680a88f8953031, + 0xabcc77118461cefc, 0xfdc20d2b36ba7c3e, + 0xd6bf94d5e57a42bc, 0x3d32907604691b4d, + 0x8637bd05af6c69b5, 0xa63f9a49c2c1b110, + 0xa7c5ac471b478423, 0xfcf80dc33721d54, + 0xd1b71758e219652b, 0xd3c36113404ea4a9, + 0x83126e978d4fdf3b, 0x645a1cac083126ea, + 0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4, + 0xcccccccccccccccc, 0xcccccccccccccccd, + 0x8000000000000000, 0x0, + 0xa000000000000000, 0x0, + 0xc800000000000000, 0x0, + 0xfa00000000000000, 0x0, + 0x9c40000000000000, 0x0, + 0xc350000000000000, 0x0, + 0xf424000000000000, 0x0, + 0x9896800000000000, 0x0, + 0xbebc200000000000, 0x0, + 0xee6b280000000000, 0x0, + 0x9502f90000000000, 0x0, + 0xba43b74000000000, 0x0, + 0xe8d4a51000000000, 0x0, + 0x9184e72a00000000, 0x0, + 0xb5e620f480000000, 0x0, + 0xe35fa931a0000000, 0x0, + 0x8e1bc9bf04000000, 0x0, + 0xb1a2bc2ec5000000, 0x0, + 0xde0b6b3a76400000, 0x0, + 0x8ac7230489e80000, 0x0, + 0xad78ebc5ac620000, 0x0, + 0xd8d726b7177a8000, 0x0, + 0x878678326eac9000, 0x0, + 0xa968163f0a57b400, 0x0, + 0xd3c21bcecceda100, 0x0, + 0x84595161401484a0, 0x0, + 0xa56fa5b99019a5c8, 0x0, + 0xcecb8f27f4200f3a, 0x0, + 0x813f3978f8940984, 0x4000000000000000, + 0xa18f07d736b90be5, 0x5000000000000000, + 0xc9f2c9cd04674ede, 0xa400000000000000, + 0xfc6f7c4045812296, 0x4d00000000000000, + 0x9dc5ada82b70b59d, 0xf020000000000000, + 0xc5371912364ce305, 0x6c28000000000000, + 0xf684df56c3e01bc6, 0xc732000000000000, + 0x9a130b963a6c115c, 0x3c7f400000000000, + 0xc097ce7bc90715b3, 0x4b9f100000000000, + 0xf0bdc21abb48db20, 0x1e86d40000000000, + 0x96769950b50d88f4, 0x1314448000000000, + 0xbc143fa4e250eb31, 0x17d955a000000000, + 0xeb194f8e1ae525fd, 0x5dcfab0800000000, + 0x92efd1b8d0cf37be, 0x5aa1cae500000000, + 0xb7abc627050305ad, 0xf14a3d9e40000000, + 0xe596b7b0c643c719, 0x6d9ccd05d0000000, + 0x8f7e32ce7bea5c6f, 0xe4820023a2000000, + 0xb35dbf821ae4f38b, 0xdda2802c8a800000, + 0xe0352f62a19e306e, 0xd50b2037ad200000, + 0x8c213d9da502de45, 0x4526f422cc340000, + 0xaf298d050e4395d6, 0x9670b12b7f410000, + 0xdaf3f04651d47b4c, 0x3c0cdd765f114000, + 0x88d8762bf324cd0f, 0xa5880a69fb6ac800, + 0xab0e93b6efee0053, 0x8eea0d047a457a00, + 0xd5d238a4abe98068, 0x72a4904598d6d880, + 0x85a36366eb71f041, 0x47a6da2b7f864750, + 0xa70c3c40a64e6c51, 0x999090b65f67d924, + 0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d, + 0x82818f1281ed449f, 0xbff8f10e7a8921a4, + 0xa321f2d7226895c7, 0xaff72d52192b6a0d, + 0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490, + 0xfee50b7025c36a08, 0x2f236d04753d5b4, + 0x9f4f2726179a2245, 0x1d762422c946590, + 0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5, + 0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2, + 0x9b934c3b330c8577, 0x63cc55f49f88eb2f, + 0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb, + 0xf316271c7fc3908a, 0x8bef464e3945ef7a, + 0x97edd871cfda3a56, 0x97758bf0e3cbb5ac, + 0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317, + 0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd, + 0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a, + 0xb975d6b6ee39e436, 0xb3e2fd538e122b44, + 0xe7d34c64a9c85d44, 0x60dbbca87196b616, + 0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd, + 0xb51d13aea4a488dd, 0x6babab6398bdbe41, + 0xe264589a4dcdab14, 0xc696963c7eed2dd1, + 0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2, + 0xb0de65388cc8ada8, 0x3b25a55f43294bcb, + 0xdd15fe86affad912, 0x49ef0eb713f39ebe, + 0x8a2dbf142dfcc7ab, 0x6e3569326c784337, + 0xacb92ed9397bf996, 0x49c2c37f07965404, + 0xd7e77a8f87daf7fb, 0xdc33745ec97be906, + 0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3, + 0xa8acd7c0222311bc, 0xc40832ea0d68ce0c, + 0xd2d80db02aabd62b, 0xf50a3fa490c30190, + 0x83c7088e1aab65db, 0x792667c6da79e0fa, + 0xa4b8cab1a1563f52, 0x577001b891185938, + 0xcde6fd5e09abcf26, 0xed4c0226b55e6f86, + 0x80b05e5ac60b6178, 0x544f8158315b05b4, + 0xa0dc75f1778e39d6, 0x696361ae3db1c721, + 0xc913936dd571c84c, 0x3bc3a19cd1e38e9, + 0xfb5878494ace3a5f, 0x4ab48a04065c723, + 0x9d174b2dcec0e47b, 0x62eb0d64283f9c76, + 0xc45d1df942711d9a, 0x3ba5d0bd324f8394, + 0xf5746577930d6500, 0xca8f44ec7ee36479, + 0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb, + 0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e, + 0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e, + 0x95d04aee3b80ece5, 0xbba1f1d158724a12, + 0xbb445da9ca61281f, 0x2a8a6e45ae8edc97, + 0xea1575143cf97226, 0xf52d09d71a3293bd, + 0x924d692ca61be758, 0x593c2626705f9c56, + 0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c, + 0xe498f455c38b997a, 0xb6dfb9c0f956447, + 0x8edf98b59a373fec, 0x4724bd4189bd5eac, + 0xb2977ee300c50fe7, 0x58edec91ec2cb657, + 0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed, + 0x8b865b215899f46c, 0xbd79e0d20082ee74, + 0xae67f1e9aec07187, 0xecd8590680a3aa11, + 0xda01ee641a708de9, 0xe80e6f4820cc9495, + 0x884134fe908658b2, 0x3109058d147fdcdd, + 0xaa51823e34a7eede, 0xbd4b46f0599fd415, + 0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a, + 0x850fadc09923329e, 0x3e2cf6bc604ddb0, + 0xa6539930bf6bff45, 0x84db8346b786151c, + 0xcfe87f7cef46ff16, 0xe612641865679a63, + 0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e, + 0xa26da3999aef7749, 0xe3be5e330f38f09d, + 0xcb090c8001ab551c, 0x5cadf5bfd3072cc5, + 0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6, + 0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa, + 0xc646d63501a1511d, 0xb281e1fd541501b8, + 0xf7d88bc24209a565, 0x1f225a7ca91a4226, + 0x9ae757596946075f, 0x3375788de9b06958, + 0xc1a12d2fc3978937, 0x52d6b1641c83ae, + 0xf209787bb47d6b84, 0xc0678c5dbd23a49a, + 0x9745eb4d50ce6332, 0xf840b7ba963646e0, + 0xbd176620a501fbff, 0xb650e5a93bc3d898, + 0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe, + 0x93ba47c980e98cdf, 0xc66f336c36b10137, + 0xb8a8d9bbe123f017, 0xb80b0047445d4184, + 0xe6d3102ad96cec1d, 0xa60dc059157491e5, + 0x9043ea1ac7e41392, 0x87c89837ad68db2f, + 0xb454e4a179dd1877, 0x29babe4598c311fb, + 0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a, + 0x8ce2529e2734bb1d, 0x1899e4a65f58660c, + 0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f, + 0xdc21a1171d42645d, 0x76707543f4fa1f73, + 0x899504ae72497eba, 0x6a06494a791c53a8, + 0xabfa45da0edbde69, 0x487db9d17636892, + 0xd6f8d7509292d603, 0x45a9d2845d3c42b6, + 0x865b86925b9bc5c2, 0xb8a2392ba45a9b2, + 0xa7f26836f282b732, 0x8e6cac7768d7141e, + 0xd1ef0244af2364ff, 0x3207d795430cd926, + 0x8335616aed761f1f, 0x7f44e6bd49e807b8, + 0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6, + 0xcd036837130890a1, 0x36dba887c37a8c0f, + 0x802221226be55a64, 0xc2494954da2c9789, + 0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c, + 0xc83553c5c8965d3d, 0x6f92829494e5acc7, + 0xfa42a8b73abbf48c, 0xcb772339ba1f17f9, + 0x9c69a97284b578d7, 0xff2a760414536efb, + 0xc38413cf25e2d70d, 0xfef5138519684aba, + 0xf46518c2ef5b8cd1, 0x7eb258665fc25d69, + 0x98bf2f79d5993802, 0xef2f773ffbd97a61, + 0xbeeefb584aff8603, 0xaafb550ffacfd8fa, + 0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38, + 0x952ab45cfa97a0b2, 0xdd945a747bf26183, + 0xba756174393d88df, 0x94f971119aeef9e4, + 0xe912b9d1478ceb17, 0x7a37cd5601aab85d, + 0x91abb422ccb812ee, 0xac62e055c10ab33a, + 0xb616a12b7fe617aa, 0x577b986b314d6009, + 0xe39c49765fdf9d94, 0xed5a7e85fda0b80b, + 0x8e41ade9fbebc27d, 0x14588f13be847307, + 0xb1d219647ae6b31c, 0x596eb2d8ae258fc8, + 0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb, + 0x8aec23d680043bee, 0x25de7bb9480d5854, + 0xada72ccc20054ae9, 0xaf561aa79a10ae6a, + 0xd910f7ff28069da4, 0x1b2ba1518094da04, + 0x87aa9aff79042286, 0x90fb44d2f05d0842, + 0xa99541bf57452b28, 0x353a1607ac744a53, + 0xd3fa922f2d1675f2, 0x42889b8997915ce8, + 0x847c9b5d7c2e09b7, 0x69956135febada11, + 0xa59bc234db398c25, 0x43fab9837e699095, + 0xcf02b2c21207ef2e, 0x94f967e45e03f4bb, + 0x8161afb94b44f57d, 0x1d1be0eebac278f5, + 0xa1ba1ba79e1632dc, 0x6462d92a69731732, + 0xca28a291859bbf93, 0x7d7b8f7503cfdcfe, + 0xfcb2cb35e702af78, 0x5cda735244c3d43e, + 0x9defbf01b061adab, 0x3a0888136afa64a7, + 0xc56baec21c7a1916, 0x88aaa1845b8fdd0, + 0xf6c69a72a3989f5b, 0x8aad549e57273d45, + 0x9a3c2087a63f6399, 0x36ac54e2f678864b, + 0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd, + 0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5, + 0x969eb7c47859e743, 0x9f644ae5a4b1b325, + 0xbc4665b596706114, 0x873d5d9f0dde1fee, + 0xeb57ff22fc0c7959, 0xa90cb506d155a7ea, + 0x9316ff75dd87cbd8, 0x9a7f12442d588f2, + 0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f, + 0xe5d3ef282a242e81, 0x8f1668c8a86da5fa, + 0x8fa475791a569d10, 0xf96e017d694487bc, + 0xb38d92d760ec4455, 0x37c981dcc395a9ac, + 0xe070f78d3927556a, 0x85bbe253f47b1417, + 0x8c469ab843b89562, 0x93956d7478ccec8e, + 0xaf58416654a6babb, 0x387ac8d1970027b2, + 0xdb2e51bfe9d0696a, 0x6997b05fcc0319e, + 0x88fcf317f22241e2, 0x441fece3bdf81f03, + 0xab3c2fddeeaad25a, 0xd527e81cad7626c3, + 0xd60b3bd56a5586f1, 0x8a71e223d8d3b074, + 0x85c7056562757456, 0xf6872d5667844e49, + 0xa738c6bebb12d16c, 0xb428f8ac016561db, + 0xd106f86e69d785c7, 0xe13336d701beba52, + 0x82a45b450226b39c, 0xecc0024661173473, + 0xa34d721642b06084, 0x27f002d7f95d0190, + 0xcc20ce9bd35c78a5, 0x31ec038df7b441f4, + 0xff290242c83396ce, 0x7e67047175a15271, + 0x9f79a169bd203e41, 0xf0062c6e984d386, + 0xc75809c42c684dd1, 0x52c07b78a3e60868, + 0xf92e0c3537826145, 0xa7709a56ccdf8a82, + 0x9bbcc7a142b17ccb, 0x88a66076400bb691, + 0xc2abf989935ddbfe, 0x6acff893d00ea435, + 0xf356f7ebf83552fe, 0x583f6b8c4124d43, + 0x98165af37b2153de, 0xc3727a337a8b704a, + 0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c, + 0xeda2ee1c7064130c, 0x1162def06f79df73, + 0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8, + 0xb9a74a0637ce2ee1, 0x6d953e2bd7173692, + 0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437, + 0x910ab1d4db9914a0, 0x1d9c9892400a22a2, + 0xb54d5e4a127f59c8, 0x2503beb6d00cab4b, + 0xe2a0b5dc971f303a, 0x2e44ae64840fd61d, + 0x8da471a9de737e24, 0x5ceaecfed289e5d2, + 0xb10d8e1456105dad, 0x7425a83e872c5f47, + 0xdd50f1996b947518, 0xd12f124e28f77719, + 0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f, + 0xace73cbfdc0bfb7b, 0x636cc64d1001550b, + 0xd8210befd30efa5a, 0x3c47f7e05401aa4e, + 0x8714a775e3e95c78, 0x65acfaec34810a71, + 0xa8d9d1535ce3b396, 0x7f1839a741a14d0d, + 0xd31045a8341ca07c, 0x1ede48111209a050, + 0x83ea2b892091e44d, 0x934aed0aab460432, + 0xa4e4b66b68b65d60, 0xf81da84d5617853f, + 0xce1de40642e3f4b9, 0x36251260ab9d668e, + 0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019, + 0xa1075a24e4421730, 0xb24cf65b8612f81f, + 0xc94930ae1d529cfc, 0xdee033f26797b627, + 0xfb9b7cd9a4a7443c, 0x169840ef017da3b1, + 0x9d412e0806e88aa5, 0x8e1f289560ee864e, + 0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2, + 0xf5b5d7ec8acb58a2, 0xae10af696774b1db, + 0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29, + 0xbff610b0cc6edd3f, 0x17fd090a58d32af3, + 0xeff394dcff8a948e, 0xddfc4b4cef07f5b0, + 0x95f83d0a1fb69cd9, 0x4abdaf101564f98e, + 0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1, + 0xea53df5fd18d5513, 0x84c86189216dc5ed, + 0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4, + 0xb7118682dbb66a77, 0x3fbc8c33221dc2a1, + 0xe4d5e82392a40515, 0xfabaf3feaa5334a, + 0x8f05b1163ba6832d, 0x29cb4d87f2a7400e, + 0xb2c71d5bca9023f8, 0x743e20e9ef511012, + 0xdf78e4b2bd342cf6, 0x914da9246b255416, + 0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e, + 0xae9672aba3d0c320, 0xa184ac2473b529b1, + 0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e, + 0x8865899617fb1871, 0x7e2fa67c7a658892, + 0xaa7eebfb9df9de8d, 0xddbb901b98feeab7, + 0xd51ea6fa85785631, 0x552a74227f3ea565, + 0x8533285c936b35de, 0xd53a88958f87275f, + 0xa67ff273b8460356, 0x8a892abaf368f137, + 0xd01fef10a657842c, 0x2d2b7569b0432d85, + 0x8213f56a67f6b29b, 0x9c3b29620e29fc73, + 0xa298f2c501f45f42, 0x8349f3ba91b47b8f, + 0xcb3f2f7642717713, 0x241c70a936219a73, + 0xfe0efb53d30dd4d7, 0xed238cd383aa0110, + 0x9ec95d1463e8a506, 0xf4363804324a40aa, + 0xc67bb4597ce2ce48, 0xb143c6053edcd0d5, + 0xf81aa16fdc1b81da, 0xdd94b7868e94050a, + 0x9b10a4e5e9913128, 0xca7cf2b4191c8326, + 0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0, + 0xf24a01a73cf2dccf, 0xbc633b39673c8cec, + 0x976e41088617ca01, 0xd5be0503e085d813, + 0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18, + 0xec9c459d51852ba2, 0xddf8e7d60ed1219e, + 0x93e1ab8252f33b45, 0xcabb90e5c942b503, + 0xb8da1662e7b00a17, 0x3d6a751f3b936243, + 0xe7109bfba19c0c9d, 0xcc512670a783ad4, + 0x906a617d450187e2, 0x27fb2b80668b24c5, + 0xb484f9dc9641e9da, 0xb1f9f660802dedf6, + 0xe1a63853bbd26451, 0x5e7873f8a0396973, + 0x8d07e33455637eb2, 0xdb0b487b6423e1e8, + 0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62, + 0xdc5c5301c56b75f7, 0x7641a140cc7810fb, + 0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d, + 0xac2820d9623bf429, 0x546345fa9fbdcd44, + 0xd732290fbacaf133, 0xa97c177947ad4095, + 0x867f59a9d4bed6c0, 0x49ed8eabcccc485d, + 0xa81f301449ee8c70, 0x5c68f256bfff5a74, + 0xd226fc195c6a2f8c, 0x73832eec6fff3111, + 0x83585d8fd9c25db7, 0xc831fd53c5ff7eab, + 0xa42e74f3d032f525, 0xba3e7ca8b77f5e55, + 0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb, + 0x80444b5e7aa7cf85, 0x7980d163cf5b81b3, + 0xa0555e361951c366, 0xd7e105bcc332621f, + 0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7, + 0xfa856334878fc150, 0xb14f98f6f0feb951, + 0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3, + 0xc3b8358109e84f07, 0xa862f80ec4700c8, + 0xf4a642e14c6262c8, 0xcd27bb612758c0fa, + 0x98e7e9cccfbd7dbd, 0x8038d51cb897789c, + 0xbf21e44003acdd2c, 0xe0470a63e6bd56c3, + 0xeeea5d5004981478, 0x1858ccfce06cac74, + 0x95527a5202df0ccb, 0xf37801e0c43ebc8, + 0xbaa718e68396cffd, 0xd30560258f54e6ba, + 0xe950df20247c83fd, 0x47c6b82ef32a2069, + 0x91d28b7416cdd27e, 0x4cdc331d57fa5441, + 0xb6472e511c81471d, 0xe0133fe4adf8e952, + 0xe3d8f9e563a198e5, 0x58180fddd97723a6, + 0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648, + }; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr uint64_t + powers_template::power_of_five_128[number_of_entries]; + +#endif + +using powers = powers_template<>; + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_DECIMAL_TO_BINARY_H +#define SIMDJSON_FASTFLOAT_DECIMAL_TO_BINARY_H + +#include +#include +#include +#include +#include +#include + +namespace simdjson_fast_float { + +// This will compute or rather approximate w * 5**q and return a pair of 64-bit +// words approximating the result, with the "high" part corresponding to the +// most significant bits and the low part corresponding to the least significant +// bits. +// +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 value128 +compute_product_approximation(int64_t q, uint64_t w) { + int const index = 2 * int(q - powers::smallest_power_of_five); + // For small values of q, e.g., q in [0,27], the answer is always exact + // because The line value128 firstproduct = full_multiplication(w, + // power_of_five_128[index]); gives the exact answer. + value128 firstproduct = + full_multiplication(w, powers::power_of_five_128[index]); + static_assert((bit_precision >= 0) && (bit_precision <= 64), + " precision should be in (0,64]"); + constexpr uint64_t precision_mask = + (bit_precision < 64) ? (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision) + : uint64_t(0xFFFFFFFFFFFFFFFF); + if ((firstproduct.high & precision_mask) == + precision_mask) { // could further guard with (lower + w < lower) + // regarding the second product, we only need secondproduct.high, but our + // expectation is that the compiler will optimize this extra work away if + // needed. + value128 secondproduct = + full_multiplication(w, powers::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { + firstproduct.high++; + } + } + return firstproduct; +} + +namespace detail { +/** + * For q in (0,350), we have that + * f = (((152170 + 65536) * q ) >> 16); + * is equal to + * floor(p) + q + * where + * p = log(5**q)/log(2) = q * log(5)/log(2) + * + * For negative values of q in (-400,0), we have that + * f = (((152170 + 65536) * q ) >> 16); + * is equal to + * -ceil(p) + q + * where + * p = log(5**-q)/log(2) = -q * log(5)/log(2) + */ +constexpr simdjson_fastfloat_really_inline int32_t power(int32_t q) noexcept { + return (((152170 + 65536) * q) >> 16) + 63; +} +} // namespace detail + +// create an adjusted mantissa, biased by the invalid power2 +// for significant digits already multiplied by 10 ** q. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 adjusted_mantissa +compute_error_scaled(int64_t q, uint64_t w, int lz) noexcept { + int hilz = int(w >> 63) ^ 1; + adjusted_mantissa answer; + answer.mantissa = w << hilz; + int bias = binary::mantissa_explicit_bits() - binary::minimum_exponent(); + answer.power2 = int32_t(detail::power(int32_t(q)) + bias - hilz - lz - 62 + + invalid_am_bias); + return answer; +} + +// w * 10 ** q, without rounding the representation up. +// the power2 in the exponent will be adjusted by invalid_am_bias. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +compute_error(int64_t q, uint64_t w) noexcept { + int lz = leading_zeroes(w); + w <<= lz; + value128 product = + compute_product_approximation(q, w); + return compute_error_scaled(q, product.high, lz); +} + +// Computers w * 10 ** q. +// The returned value should be a valid number that simply needs to be +// packed. However, in some very rare cases, the computation will fail. In such +// cases, we return an adjusted_mantissa with a negative power of 2: the caller +// should recompute in such cases. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +compute_float(int64_t q, uint64_t w) noexcept { + adjusted_mantissa answer; + if ((w == 0) || (q < binary::smallest_power_of_ten())) { + answer.power2 = 0; + answer.mantissa = 0; + // result should be zero + return answer; + } + if (q > binary::largest_power_of_ten()) { + // we want to get infinity: + answer.power2 = binary::infinite_power(); + answer.mantissa = 0; + return answer; + } + // At this point in time q is in [powers::smallest_power_of_five, + // powers::largest_power_of_five]. + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(w); + w <<= lz; + + // The required precision is binary::mantissa_explicit_bits() + 3 because + // 1. We need the implicit bit + // 2. We need an extra bit for rounding purposes + // 3. We might lose a bit due to the "upperbit" routine (result too small, + // requiring a shift) + + value128 product = + compute_product_approximation(q, w); + // The computed 'product' is always sufficient. + // Mathematical proof: + // Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback (to + // appear) See script/mushtak_lemire.py + + // The "compute_product_approximation" function can be slightly slower than a + // branchless approach: value128 product = compute_product(q, w); but in + // practice, we can win big with the compute_product_approximation if its + // additional branch is easily predicted. Which is best is data specific. + int upperbit = int(product.high >> 63); + int shift = upperbit + 64 - binary::mantissa_explicit_bits() - 3; + + answer.mantissa = product.high >> shift; + + answer.power2 = int32_t(detail::power(int32_t(q)) + upperbit - lz - + binary::minimum_exponent()); + if (answer.power2 <= 0) { // we have a subnormal? + // Here have that answer.power2 <= 0 so -answer.power2 >= 0 + if (-answer.power2 + 1 >= + 64) { // if we have more than 64 bits below the minimum exponent, you + // have a zero for sure. + answer.power2 = 0; + answer.mantissa = 0; + // result should be zero + return answer; + } + // next line is safe because -answer.power2 + 1 < 64 + answer.mantissa >>= -answer.power2 + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0 in the 32-bit and + // and 64-bit case (with no more than 19 digits). + answer.mantissa += (answer.mantissa & 1); // round up + answer.mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + answer.power2 = + (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) + ? 0 + : 1; + return answer; + } + + // usually, we round *up*, but if we fall right in between and and we have an + // even basis, we need to round down + // We are only concerned with the cases where 5**q fits in single 64-bit word. + if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) && + (q <= binary::max_exponent_round_to_even()) && + ((answer.mantissa & 3) == 1)) { // we may fall between two floats! + // To be in-between two floats we need that in doing + // answer.mantissa = product.high >> (upperbit + 64 - + // binary::mantissa_explicit_bits() - 3); + // ... we dropped out only zeroes. But if this happened, then we can go + // back!!! + if ((answer.mantissa << shift) == product.high) { + answer.mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + answer.mantissa += (answer.mantissa & 1); // round up + answer.mantissa >>= 1; + if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) { + answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits()); + answer.power2++; // undo previous addition + } + + answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits()); + if (answer.power2 >= binary::infinite_power()) { // infinity + answer.power2 = binary::infinite_power(); + answer.mantissa = 0; + } + return answer; +} + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_BIGINT_H +#define SIMDJSON_FASTFLOAT_BIGINT_H + +#include +#include +#include +#include + + +namespace simdjson_fast_float { + +// the limb width: we want efficient multiplication of double the bits in +// limb, or for 64-bit limbs, at least 64-bit multiplication where we can +// extract the high and low parts efficiently. this is every 64-bit +// architecture except for sparc, which emulates 128-bit multiplication. +// we might have platforms where `CHAR_BIT` is not 8, so let's avoid +// doing `8 * sizeof(limb)`. +#if defined(SIMDJSON_FASTFLOAT_64BIT) && !defined(__sparc) +#define SIMDJSON_FASTFLOAT_64BIT_LIMB 1 +typedef uint64_t limb; +constexpr size_t limb_bits = 64; +#else +#define SIMDJSON_FASTFLOAT_32BIT_LIMB +typedef uint32_t limb; +constexpr size_t limb_bits = 32; +#endif + +typedef span limb_span; + +// number of bits in a bigint. this needs to be at least the number +// of bits required to store the largest bigint, which is +// `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or +// ~3600 bits, so we round to 4000. +constexpr size_t bigint_bits = 4000; +constexpr size_t bigint_limbs = bigint_bits / limb_bits; + +// vector-like type that is allocated on the stack. the entire +// buffer is pre-allocated, and only the length changes. +template struct stackvec { + limb data[size]; + // we never need more than 150 limbs + uint16_t length{0}; + + stackvec() = default; + stackvec(stackvec const &) = delete; + stackvec &operator=(stackvec const &) = delete; + stackvec(stackvec &&) = delete; + stackvec &operator=(stackvec &&other) = delete; + + // create stack vector from existing limb span. + SIMDJSON_FASTFLOAT_CONSTEXPR20 stackvec(limb_span s) { + SIMDJSON_FASTFLOAT_ASSERT(try_extend(s)); + } + + SIMDJSON_FASTFLOAT_CONSTEXPR14 limb &operator[](size_t index) noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + return data[index]; + } + + SIMDJSON_FASTFLOAT_CONSTEXPR14 const limb &operator[](size_t index) const noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + return data[index]; + } + + // index from the end of the container + SIMDJSON_FASTFLOAT_CONSTEXPR14 const limb &rindex(size_t index) const noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + size_t rindex = length - index - 1; + return data[rindex]; + } + + // set the length, without bounds checking. + SIMDJSON_FASTFLOAT_CONSTEXPR14 void set_len(size_t len) noexcept { + length = uint16_t(len); + } + + constexpr size_t len() const noexcept { return length; } + + constexpr bool is_empty() const noexcept { return length == 0; } + + constexpr size_t capacity() const noexcept { return size; } + + // append item to vector, without bounds checking + SIMDJSON_FASTFLOAT_CONSTEXPR14 void push_unchecked(limb value) noexcept { + data[length] = value; + length++; + } + + // append item to vector, returning if item was added + SIMDJSON_FASTFLOAT_CONSTEXPR14 bool try_push(limb value) noexcept { + if (len() < capacity()) { + push_unchecked(value); + return true; + } else { + return false; + } + } + + // add items to the vector, from a span, without bounds checking + SIMDJSON_FASTFLOAT_CONSTEXPR20 void extend_unchecked(limb_span s) noexcept { + limb *ptr = data + length; + std::copy_n(s.ptr, s.len(), ptr); + set_len(len() + s.len()); + } + + // try to add items to the vector, returning if items were added + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool try_extend(limb_span s) noexcept { + if (len() + s.len() <= capacity()) { + extend_unchecked(s); + return true; + } else { + return false; + } + } + + // resize the vector, without bounds checking + // if the new size is longer than the vector, assign value to each + // appended item. + SIMDJSON_FASTFLOAT_CONSTEXPR20 + void resize_unchecked(size_t new_len, limb value) noexcept { + if (new_len > len()) { + size_t count = new_len - len(); + limb *first = data + len(); + limb *last = first + count; + ::std::fill(first, last, value); + set_len(new_len); + } else { + set_len(new_len); + } + } + + // try to resize the vector, returning if the vector was resized. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool try_resize(size_t new_len, limb value) noexcept { + if (new_len > capacity()) { + return false; + } else { + resize_unchecked(new_len, value); + return true; + } + } + + // check if any limbs are non-zero after the given index. + // this needs to be done in reverse order, since the index + // is relative to the most significant limbs. + SIMDJSON_FASTFLOAT_CONSTEXPR14 bool nonzero(size_t index) const noexcept { + while (index < len()) { + if (rindex(index) != 0) { + return true; + } + index++; + } + return false; + } + + // normalize the big integer, so most-significant zero limbs are removed. + SIMDJSON_FASTFLOAT_CONSTEXPR14 void normalize() noexcept { + while (len() > 0 && rindex(0) == 0) { + length--; + } + } +}; + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t +empty_hi64(bool &truncated) noexcept { + truncated = false; + return 0; +} - std::memmove(buf + (2 + static_cast(-n)), buf, - static_cast(k)); - buf[0] = '0'; - buf[1] = '.'; - std::memset(buf + 2, '0', static_cast(-n)); - return buf + (2U + static_cast(-n) + static_cast(k)); +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint64_hi64(uint64_t r0, bool &truncated) noexcept { + truncated = false; + int shl = leading_zeroes(r0); + return r0 << shl; +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept { + int shl = leading_zeroes(r0); + if (shl == 0) { + truncated = r1 != 0; + return r0; + } else { + int shr = 64 - shl; + truncated = (r1 << shl) != 0; + return (r0 << shl) | (r1 >> shr); } +} - if (k == 1) { - // dE+123 - // len <= 1 + 5 +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint32_hi64(uint32_t r0, bool &truncated) noexcept { + return uint64_hi64(r0, truncated); +} - buf += 1; - } else { - // d.igitsE+123 - // len <= max_digits10 + 1 + 5 +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint32_hi64(uint32_t r0, uint32_t r1, bool &truncated) noexcept { + uint64_t x0 = r0; + uint64_t x1 = r1; + return uint64_hi64((x0 << 32) | x1, truncated); +} - std::memmove(buf + 2, buf + 1, static_cast(k) - 1); - buf[1] = '.'; - buf += 1 + static_cast(k); +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept { + uint64_t x0 = r0; + uint64_t x1 = r1; + uint64_t x2 = r2; + return uint64_hi64(x0, (x1 << 32) | x2, truncated); +} + +// add two small integers, checking for overflow. +// we want an efficient operation. for msvc, where +// we don't have built-in intrinsics, this is still +// pretty fast. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 limb +scalar_add(limb x, limb y, bool &overflow) noexcept { + limb z; +// gcc and clang +#if defined(__has_builtin) +#if __has_builtin(__builtin_add_overflow) + if (!cpp20_and_in_constexpr()) { + overflow = __builtin_add_overflow(x, y, &z); + return z; } +#endif +#endif - *buf++ = 'e'; - return append_exponent(buf, n - 1); + // generic, this still optimizes correctly on MSVC. + z = x + y; + overflow = z < x; + return z; +} + +// multiply two small integers, getting both the high and low bits. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 limb +scalar_mul(limb x, limb y, limb &carry) noexcept { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB +#if defined(__SIZEOF_INT128__) + // GCC and clang both define it as an extension. + __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carry); + carry = limb(z >> limb_bits); + return limb(z); +#else + // fallback, no native 128-bit integer multiplication with carry. + // on msvc, this optimizes identically, somehow. + value128 z = full_multiplication(x, y); + bool overflow; + z.low = scalar_add(z.low, carry, overflow); + z.high += uint64_t(overflow); // cannot overflow + carry = z.high; + return z.low; +#endif +#else + uint64_t z = uint64_t(x) * uint64_t(y) + uint64_t(carry); + carry = limb(z >> limb_bits); + return limb(z); +#endif } -} // namespace dtoa_impl +// add scalar value to bigint starting from offset. +// used in grade school multiplication +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool small_add_from(stackvec &vec, limb y, + size_t start) noexcept { + size_t index = start; + limb carry = y; + bool overflow; + while (carry != 0 && index < vec.len()) { + vec[index] = scalar_add(vec[index], carry, overflow); + carry = limb(overflow); + index += 1; + } + if (carry != 0) { + SIMDJSON_FASTFLOAT_TRY(vec.try_push(carry)); + } + return true; +} -/*! -The format of the resulting decimal representation is similar to printf's %g -format. Returns an iterator pointing past-the-end of the decimal representation. -@note The input number must be finite, i.e. NaN's and Inf's are not supported. -@note The buffer must be large enough. -@note The result is NOT null-terminated. -*/ -char *to_chars(char *first, const char *last, double value) { - static_cast(last); // maybe unused - fix warning - bool negative = std::signbit(value); - if (negative) { - value = -value; - *first++ = '-'; +// add scalar value to bigint. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +small_add(stackvec &vec, limb y) noexcept { + return small_add_from(vec, y, 0); +} + +// multiply bigint by scalar value. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool small_mul(stackvec &vec, + limb y) noexcept { + limb carry = 0; + for (size_t index = 0; index < vec.len(); index++) { + vec[index] = scalar_mul(vec[index], y, carry); + } + if (carry != 0) { + SIMDJSON_FASTFLOAT_TRY(vec.try_push(carry)); } + return true; +} - if (value == 0) // +-0 - { - *first++ = '0'; - // Make it look like a floating-point number (#362, #378) - *first++ = '.'; - *first++ = '0'; - return first; +// add bigint to bigint starting from index. +// used in grade school multiplication +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 bool large_add_from(stackvec &x, limb_span y, + size_t start) noexcept { + // the effective x buffer is from `xstart..x.len()`, so exit early + // if we can't get that current range. + if (x.len() < start || y.len() > x.len() - start) { + SIMDJSON_FASTFLOAT_TRY(x.try_resize(y.len() + start, 0)); } - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e. the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::grisu2(first, len, decimal_exponent, value); - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - constexpr int kMaxExp = std::numeric_limits::digits10; - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, - kMaxExp); + bool carry = false; + for (size_t index = 0; index < y.len(); index++) { + limb xi = x[index + start]; + limb yi = y[index]; + bool c1 = false; + bool c2 = false; + xi = scalar_add(xi, yi, c1); + if (carry) { + xi = scalar_add(xi, 1, c2); + } + x[index + start] = xi; + carry = c1 | c2; + } + + // handle overflow + if (carry) { + SIMDJSON_FASTFLOAT_TRY(small_add_from(x, 1, y.len() + start)); + } + return true; } -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_SRC_TO_CHARS_CPP -/* end file to_chars.cpp */ -/* including from_chars.cpp: #include */ -/* begin file from_chars.cpp */ -#ifndef SIMDJSON_SRC_FROM_CHARS_CPP -#define SIMDJSON_SRC_FROM_CHARS_CPP +// add bigint to bigint. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +large_add_from(stackvec &x, limb_span y) noexcept { + return large_add_from(x, y, 0); +} + +// grade-school multiplication algorithm +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 bool long_mul(stackvec &x, limb_span y) noexcept { + limb_span xs = limb_span(x.data, x.len()); + stackvec z(xs); + limb_span zs = limb_span(z.data, z.len()); + + if (y.len() != 0) { + limb y0 = y[0]; + SIMDJSON_FASTFLOAT_TRY(small_mul(x, y0)); + for (size_t index = 1; index < y.len(); index++) { + limb yi = y[index]; + stackvec zi; + if (yi != 0) { + // re-use the same buffer throughout + zi.set_len(0); + SIMDJSON_FASTFLOAT_TRY(zi.try_extend(zs)); + SIMDJSON_FASTFLOAT_TRY(small_mul(zi, yi)); + limb_span zis = limb_span(zi.data, zi.len()); + SIMDJSON_FASTFLOAT_TRY(large_add_from(x, zis, index)); + } + } + } -/* skipped duplicate #include */ + x.normalize(); + return true; +} -#include -#include -#include +// grade-school multiplication algorithm +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 bool large_mul(stackvec &x, limb_span y) noexcept { + if (y.len() == 1) { + SIMDJSON_FASTFLOAT_TRY(small_mul(x, y[0])); + } else { + SIMDJSON_FASTFLOAT_TRY(long_mul(x, y)); + } + return true; +} -namespace simdjson { -namespace internal { +template struct pow5_tables { + static constexpr uint32_t large_step = 135; + static constexpr uint64_t small_power_of_5[] = { + 1UL, + 5UL, + 25UL, + 125UL, + 625UL, + 3125UL, + 15625UL, + 78125UL, + 390625UL, + 1953125UL, + 9765625UL, + 48828125UL, + 244140625UL, + 1220703125UL, + 6103515625UL, + 30517578125UL, + 152587890625UL, + 762939453125UL, + 3814697265625UL, + 19073486328125UL, + 95367431640625UL, + 476837158203125UL, + 2384185791015625UL, + 11920928955078125UL, + 59604644775390625UL, + 298023223876953125UL, + 1490116119384765625UL, + 7450580596923828125UL, + }; +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + constexpr static limb large_power_of_5[] = { + 1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL, + 10482974169319127550UL, 198276706040285095UL}; +#else + constexpr static limb large_power_of_5[] = { + 4279965485U, 329373468U, 4020270615U, 2137533757U, 4287402176U, + 1057042919U, 1071430142U, 2440757623U, 381945767U, 46164893U}; +#endif +}; -/** - * The code in the internal::from_chars function is meant to handle the floating-point number parsing - * when we have more than 19 digits in the decimal mantissa. This should only be seen - * in adversarial scenarios: we do not expect production systems to even produce - * such floating-point numbers. - * - * The parser is based on work by Nigel Tao (at https://github.com/google/wuffs/) - * who credits Ken Thompson for the design (via a reference to the Go source - * code). See - * https://github.com/google/wuffs/blob/aa46859ea40c72516deffa1b146121952d6dfd3b/internal/cgen/base/floatconv-submodule-data.c - * https://github.com/google/wuffs/blob/46cd8105f47ca07ae2ba8e6a7818ef9c0df6c152/internal/cgen/base/floatconv-submodule-code.c - * It is probably not very fast but it is a fallback that should almost never be - * called in real life. Google Wuffs is published under APL 2.0. - **/ +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE -namespace { -constexpr uint32_t max_digits = 768; -constexpr int32_t decimal_point_range = 2047; -} // namespace +template constexpr uint32_t pow5_tables::large_step; -struct adjusted_mantissa { - uint64_t mantissa; - int power2; - adjusted_mantissa() : mantissa(0), power2(0) {} -}; +template constexpr uint64_t pow5_tables::small_power_of_5[]; -struct decimal { - uint32_t num_digits; - int32_t decimal_point; - bool negative; - bool truncated; - uint8_t digits[max_digits]; -}; +template constexpr limb pow5_tables::large_power_of_5[]; -template struct binary_format { - static constexpr int mantissa_explicit_bits(); - static constexpr int minimum_exponent(); - static constexpr int infinite_power(); - static constexpr int sign_index(); -}; +#endif -template <> constexpr int binary_format::mantissa_explicit_bits() { - return 52; -} +// big integer type. implements a small subset of big integer +// arithmetic, using simple algorithms since asymptotically +// faster algorithms are slower for a small number of limbs. +// all operations assume the big-integer is normalized. +struct bigint : pow5_tables<> { + // storage of the limbs, in little-endian order. + stackvec vec; -template <> constexpr int binary_format::minimum_exponent() { - return -1023; -} -template <> constexpr int binary_format::infinite_power() { - return 0x7FF; -} + SIMDJSON_FASTFLOAT_CONSTEXPR20 bigint() : vec() {} -template <> constexpr int binary_format::sign_index() { return 63; } + bigint(bigint const &) = delete; + bigint &operator=(bigint const &) = delete; + bigint(bigint &&) = delete; + bigint &operator=(bigint &&other) = delete; -bool is_integer(char c) noexcept { return (c >= '0' && c <= '9'); } + SIMDJSON_FASTFLOAT_CONSTEXPR20 bigint(uint64_t value) : vec() { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + vec.push_unchecked(value); +#else + vec.push_unchecked(uint32_t(value)); + vec.push_unchecked(uint32_t(value >> 32)); +#endif + vec.normalize(); + } -// This should always succeed since it follows a call to parse_number. -decimal parse_decimal(const char *&p) noexcept { - decimal answer; - answer.num_digits = 0; - answer.decimal_point = 0; - answer.truncated = false; - answer.negative = (*p == '-'); - if ((*p == '-') || (*p == '+')) { - ++p; + // get the high 64 bits from the vector, and if bits were truncated. + // this is to get the significant digits for the float. + SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t hi64(bool &truncated) const noexcept { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + if (vec.len() == 0) { + return empty_hi64(truncated); + } else if (vec.len() == 1) { + return uint64_hi64(vec.rindex(0), truncated); + } else { + uint64_t result = uint64_hi64(vec.rindex(0), vec.rindex(1), truncated); + truncated |= vec.nonzero(2); + return result; + } +#else + if (vec.len() == 0) { + return empty_hi64(truncated); + } else if (vec.len() == 1) { + return uint32_hi64(vec.rindex(0), truncated); + } else if (vec.len() == 2) { + return uint32_hi64(vec.rindex(0), vec.rindex(1), truncated); + } else { + uint64_t result = + uint32_hi64(vec.rindex(0), vec.rindex(1), vec.rindex(2), truncated); + truncated |= vec.nonzero(3); + return result; + } +#endif } - while (*p == '0') { - ++p; + // compare two big integers, returning the large value. + // assumes both are normalized. if the return value is + // negative, other is larger, if the return value is + // positive, this is larger, otherwise they are equal. + // the limbs are stored in little-endian order, so we + // must compare the limbs in ever order. + SIMDJSON_FASTFLOAT_CONSTEXPR20 int compare(bigint const &other) const noexcept { + if (vec.len() > other.vec.len()) { + return 1; + } else if (vec.len() < other.vec.len()) { + return -1; + } else { + for (size_t index = vec.len(); index > 0; index--) { + limb xi = vec[index - 1]; + limb yi = other.vec[index - 1]; + if (xi > yi) { + return 1; + } else if (xi < yi) { + return -1; + } + } + return 0; + } } - while (is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); + + // shift left each limb n bits, carrying over to the new limb + // returns true if we were able to shift all the digits. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl_bits(size_t n) noexcept { + // Internally, for each item, we shift left by n, and add the previous + // right shifted limb-bits. + // For example, we transform (for u8) shifted left 2, to: + // b10100100 b01000010 + // b10 b10010001 b00001000 + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n != 0); + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n < sizeof(limb) * 8); + + size_t shl = n; + size_t shr = limb_bits - shl; + limb prev = 0; + for (size_t index = 0; index < vec.len(); index++) { + limb xi = vec[index]; + vec[index] = (xi << shl) | (prev >> shr); + prev = xi; } - answer.num_digits++; - ++p; + + limb carry = prev >> shr; + if (carry != 0) { + return vec.try_push(carry); + } + return true; } - if (*p == '.') { - ++p; - const char *first_after_period = p; - // if we have not yet encountered a zero, we have to skip it as well - if (answer.num_digits == 0) { - // skip zeros - while (*p == '0') { - ++p; - } + + // move the limbs left by `n` limbs. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl_limbs(size_t n) noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n != 0); + if (n + vec.len() > vec.capacity()) { + return false; + } else if (!vec.is_empty()) { + // move limbs + limb *dst = vec.data + n; + limb const *src = vec.data; + std::copy_backward(src, src + vec.len(), dst + vec.len()); + // fill in empty limbs + limb *first = vec.data; + limb *last = first + n; + ::std::fill(first, last, 0); + vec.set_len(n + vec.len()); + return true; + } else { + return true; } - while (is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); - } - answer.num_digits++; - ++p; + } + + // move the limbs left by `n` bits. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl(size_t n) noexcept { + size_t rem = n % limb_bits; + size_t div = n / limb_bits; + if (rem != 0) { + SIMDJSON_FASTFLOAT_TRY(shl_bits(rem)); } - answer.decimal_point = int32_t(first_after_period - p); + if (div != 0) { + SIMDJSON_FASTFLOAT_TRY(shl_limbs(div)); + } + return true; } - if(answer.num_digits > 0) { - const char *preverse = p - 1; - int32_t trailing_zeros = 0; - while ((*preverse == '0') || (*preverse == '.')) { - if(*preverse == '0') { trailing_zeros++; }; - --preverse; + + // get the number of leading zeros in the bigint. + SIMDJSON_FASTFLOAT_CONSTEXPR20 int ctlz() const noexcept { + if (vec.is_empty()) { + return 0; + } else { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + return leading_zeroes(vec.rindex(0)); +#else + // no use defining a specialized leading_zeroes for a 32-bit type. + uint64_t r0 = vec.rindex(0); + return leading_zeroes(r0 << 32); +#endif } - answer.decimal_point += int32_t(answer.num_digits); - answer.num_digits -= uint32_t(trailing_zeros); } - if(answer.num_digits > max_digits ) { - answer.num_digits = max_digits; - answer.truncated = true; + + // get the number of bits in the bigint. + SIMDJSON_FASTFLOAT_CONSTEXPR20 int bit_length() const noexcept { + int lz = ctlz(); + return int(limb_bits * vec.len()) - lz; } - if (('e' == *p) || ('E' == *p)) { - ++p; - bool neg_exp = false; - if ('-' == *p) { - neg_exp = true; - ++p; - } else if ('+' == *p) { - ++p; + + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool mul(limb y) noexcept { return small_mul(vec, y); } + + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool add(limb y) noexcept { return small_add(vec, y); } + + // multiply as if by 2 raised to a power. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow2(uint32_t exp) noexcept { return shl(exp); } + + // multiply as if by 5 raised to a power. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow5(uint32_t exp) noexcept { + // multiply by a power of 5 + size_t large_length = sizeof(large_power_of_5) / sizeof(limb); + limb_span large = limb_span(large_power_of_5, large_length); + while (exp >= large_step) { + SIMDJSON_FASTFLOAT_TRY(large_mul(vec, large)); + exp -= large_step; } - int32_t exp_number = 0; // exponential part - while (is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - if (exp_number < 0x10000) { - exp_number = 10 * exp_number + digit; - } - ++p; +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + uint32_t small_step = 27; + limb max_native = 7450580596923828125UL; +#else + uint32_t small_step = 13; + limb max_native = 1220703125U; +#endif + while (exp >= small_step) { + SIMDJSON_FASTFLOAT_TRY(small_mul(vec, max_native)); + exp -= small_step; + } + if (exp != 0) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + // This is similar to https://github.com/llvm/llvm-project/issues/47746, + // except the workaround described there don't work here + SIMDJSON_FASTFLOAT_TRY(small_mul(vec, limb((static_cast(small_power_of_5[0]), + small_power_of_5[exp])))); } - answer.decimal_point += (neg_exp ? -exp_number : exp_number); + + return true; } - return answer; + + // multiply as if by 10 raised to a power. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow10(uint32_t exp) noexcept { + SIMDJSON_FASTFLOAT_TRY(pow5(exp)); + return pow2(exp); + } +}; + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_DIGIT_COMPARISON_H +#define SIMDJSON_FASTFLOAT_DIGIT_COMPARISON_H + +#include +#include +#include + + +namespace simdjson_fast_float { + +// 1e0 to 1e19 +constexpr static uint64_t powers_of_ten_uint64[] = {1UL, + 10UL, + 100UL, + 1000UL, + 10000UL, + 100000UL, + 1000000UL, + 10000000UL, + 100000000UL, + 1000000000UL, + 10000000000UL, + 100000000000UL, + 1000000000000UL, + 10000000000000UL, + 100000000000000UL, + 1000000000000000UL, + 10000000000000000UL, + 100000000000000000UL, + 1000000000000000000UL, + 10000000000000000000UL}; + +// calculate the exponent, in scientific notation, of the number. +// this algorithm is not even close to optimized, but it has no practical +// effect on performance: in order to have a faster algorithm, we'd need +// to slow down performance for faster algorithms, and this is still fast. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int32_t +scientific_exponent(uint64_t mantissa, int32_t exponent) noexcept { + while (mantissa >= 10000) { + mantissa /= 10000; + exponent += 4; + } + while (mantissa >= 100) { + mantissa /= 100; + exponent += 2; + } + while (mantissa >= 10) { + mantissa /= 10; + exponent += 1; + } + return exponent; +} + +// this converts a native floating-point number to an extended-precision float. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +to_extended(T value) noexcept { + using equiv_uint = equiv_uint_t; + constexpr equiv_uint exponent_mask = binary_format::exponent_mask(); + constexpr equiv_uint mantissa_mask = binary_format::mantissa_mask(); + constexpr equiv_uint hidden_bit_mask = binary_format::hidden_bit_mask(); + + adjusted_mantissa am; + int32_t bias = binary_format::mantissa_explicit_bits() - + binary_format::minimum_exponent(); + equiv_uint bits; +#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST + bits = std::bit_cast(value); +#else + ::memcpy(&bits, &value, sizeof(T)); +#endif + if ((bits & exponent_mask) == 0) { + // denormal + am.power2 = 1 - bias; + am.mantissa = bits & mantissa_mask; + } else { + // normal + am.power2 = int32_t((bits & exponent_mask) >> + binary_format::mantissa_explicit_bits()); + am.power2 -= bias; + am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; + } + + return am; } -// This should always succeed since it follows a call to parse_number. -// Will not read at or beyond the "end" pointer. -decimal parse_decimal(const char *&p, const char * end) noexcept { - decimal answer; - answer.num_digits = 0; - answer.decimal_point = 0; - answer.truncated = false; - if(p == end) { return answer; } // should never happen - answer.negative = (*p == '-'); - if ((*p == '-') || (*p == '+')) { - ++p; +// get the extended precision value of the halfway point between b and b+u. +// we are given a native float that represents b, so we need to adjust it +// halfway between b and b+u. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +to_extended_halfway(T value) noexcept { + adjusted_mantissa am = to_extended(value); + am.mantissa <<= 1; + am.mantissa += 1; + am.power2 -= 1; + return am; +} + +// round an extended-precision float to the nearest machine float. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am, + callback cb) noexcept { + int32_t mantissa_shift = 64 - binary_format::mantissa_explicit_bits() - 1; + if (-am.power2 >= mantissa_shift) { + // have a denormal float + int32_t shift = -am.power2 + 1; + cb(am, (shift < 64 ? shift : 64)); + // check for round-up: if rounding-nearest carried us to the hidden bit. + am.power2 = (am.mantissa < + (uint64_t(1) << binary_format::mantissa_explicit_bits())) + ? 0 + : 1; + return; } - while ((p != end) && (*p == '0')) { - ++p; + // have a normal float, use the default shift. + cb(am, mantissa_shift); + + // check for carry + if (am.mantissa >= + (uint64_t(2) << binary_format::mantissa_explicit_bits())) { + am.mantissa = (uint64_t(1) << binary_format::mantissa_explicit_bits()); + am.power2++; + } + + // check for infinite: we could have carried to an infinite power + am.mantissa &= ~(uint64_t(1) << binary_format::mantissa_explicit_bits()); + if (am.power2 >= binary_format::infinite_power()) { + am.power2 = binary_format::infinite_power(); + am.mantissa = 0; + } +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void +round_nearest_tie_even(adjusted_mantissa &am, int32_t shift, + callback cb) noexcept { + uint64_t const mask = (shift == 64) ? UINT64_MAX : (uint64_t(1) << shift) - 1; + uint64_t const halfway = (shift == 0) ? 0 : uint64_t(1) << (shift - 1); + uint64_t truncated_bits = am.mantissa & mask; + bool is_above = truncated_bits > halfway; + bool is_halfway = truncated_bits == halfway; + + // shift digits into position + if (shift == 64) { + am.mantissa = 0; + } else { + am.mantissa >>= shift; + } + am.power2 += shift; + + bool is_odd = (am.mantissa & 1) == 1; + am.mantissa += uint64_t(cb(is_odd, is_halfway, is_above)); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void +round_down(adjusted_mantissa &am, int32_t shift) noexcept { + if (shift == 64) { + am.mantissa = 0; + } else { + am.mantissa >>= shift; } - while ((p != end) && is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); + am.power2 += shift; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +skip_zeros(UC const *&first, UC const *last) noexcept { + uint64_t val; + while (!cpp20_and_in_constexpr() && + std::distance(first, last) >= int_cmp_len()) { + ::memcpy(&val, first, sizeof(uint64_t)); + if (val != int_cmp_zeros()) { + break; } - answer.num_digits++; - ++p; + first += int_cmp_len(); } - if ((p != end) && (*p == '.')) { - ++p; - if(p == end) { return answer; } // should never happen - const char *first_after_period = p; - // if we have not yet encountered a zero, we have to skip it as well - if (answer.num_digits == 0) { - // skip zeros - while (*p == '0') { - ++p; - } + while (first != last) { + if (*first != UC('0')) { + break; } - while ((p != end) && is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); - } - answer.num_digits++; - ++p; + first++; + } +} + +// determine if any non-zero digits were truncated. +// all characters must be valid digits. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +is_truncated(UC const *first, UC const *last) noexcept { + // do 8-bit optimizations, can just compare to 8 literal 0s. + uint64_t val; + while (!cpp20_and_in_constexpr() && + std::distance(first, last) >= int_cmp_len()) { + ::memcpy(&val, first, sizeof(uint64_t)); + if (val != int_cmp_zeros()) { + return true; } - answer.decimal_point = int32_t(first_after_period - p); + first += int_cmp_len(); } - if(answer.num_digits > 0) { - const char *preverse = p - 1; - int32_t trailing_zeros = 0; - while ((*preverse == '0') || (*preverse == '.')) { - if(*preverse == '0') { trailing_zeros++; }; - --preverse; + while (first != last) { + if (*first != UC('0')) { + return true; } - answer.decimal_point += int32_t(answer.num_digits); - answer.num_digits -= uint32_t(trailing_zeros); + ++first; } - if(answer.num_digits > max_digits ) { - answer.num_digits = max_digits; - answer.truncated = true; + return false; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +is_truncated(span s) noexcept { + return is_truncated(s.ptr, s.ptr + s.len()); +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +parse_eight_digits(UC const *&p, limb &value, size_t &counter, + size_t &count) noexcept { + value = value * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + counter += 8; + count += 8; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void +parse_one_digit(UC const *&p, limb &value, size_t &counter, + size_t &count) noexcept { + value = value * 10 + limb(*p - UC('0')); + p++; + counter++; + count++; +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +add_native(bigint &big, limb power, limb value) noexcept { + big.mul(power); + big.add(value); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +round_up_bigint(bigint &big, size_t &count) noexcept { + // need to round-up the digits, but need to avoid rounding + // ....9999 to ...10000, which could cause a false halfway point. + add_native(big, 10, 1); + count++; +} + +// parse the significant digits into a big integer +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +parse_mantissa(bigint &result, parsed_number_string_t &num, + size_t max_digits, size_t &digits) noexcept { + // try to minimize the number of big integer and scalar multiplication. + // therefore, try to parse 8 digits at a time, and multiply by the largest + // scalar value (9 or 19 digits) for each step. + size_t counter = 0; + digits = 0; + limb value = 0; +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + size_t step = 19; +#else + size_t step = 9; +#endif + + // process all integer digits. + UC const *p = num.integer.ptr; + UC const *pend = p + num.integer.len(); + skip_zeros(p, pend); + // process all digits, in increments of step per loop + while (p != pend) { + while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && + (max_digits - digits >= 8)) { + parse_eight_digits(p, value, counter, digits); + } + while (counter < step && p != pend && digits < max_digits) { + parse_one_digit(p, value, counter, digits); + } + if (digits == max_digits) { + // add the temporary value, then check if we've truncated any digits + add_native(result, limb(powers_of_ten_uint64[counter]), value); + bool truncated = is_truncated(p, pend); + if (num.fraction.ptr != nullptr) { + truncated |= is_truncated(num.fraction); + } + if (truncated) { + round_up_bigint(result, digits); + } + return; + } else { + add_native(result, limb(powers_of_ten_uint64[counter]), value); + counter = 0; + value = 0; + } } - if ((p != end) && (('e' == *p) || ('E' == *p))) { - ++p; - if(p == end) { return answer; } // should never happen - bool neg_exp = false; - if ('-' == *p) { - neg_exp = true; - ++p; - } else if ('+' == *p) { - ++p; + + // add our fraction digits, if they're available. + if (num.fraction.ptr != nullptr) { + p = num.fraction.ptr; + pend = p + num.fraction.len(); + if (digits == 0) { + skip_zeros(p, pend); } - int32_t exp_number = 0; // exponential part - while ((p != end) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - if (exp_number < 0x10000) { - exp_number = 10 * exp_number + digit; + // process all digits, in increments of step per loop + while (p != pend) { + while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && + (max_digits - digits >= 8)) { + parse_eight_digits(p, value, counter, digits); + } + while (counter < step && p != pend && digits < max_digits) { + parse_one_digit(p, value, counter, digits); + } + if (digits == max_digits) { + // add the temporary value, then check if we've truncated any digits + add_native(result, limb(powers_of_ten_uint64[counter]), value); + bool truncated = is_truncated(p, pend); + if (truncated) { + round_up_bigint(result, digits); + } + return; + } else { + add_native(result, limb(powers_of_ten_uint64[counter]), value); + counter = 0; + value = 0; } - ++p; } - answer.decimal_point += (neg_exp ? -exp_number : exp_number); } + + if (counter != 0) { + add_native(result, limb(powers_of_ten_uint64[counter]), value); + } +} + +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +positive_digit_comp(bigint &bigmant, int32_t exponent) noexcept { + SIMDJSON_FASTFLOAT_ASSERT(bigmant.pow10(uint32_t(exponent))); + adjusted_mantissa answer; + bool truncated; + answer.mantissa = bigmant.hi64(truncated); + int bias = binary_format::mantissa_explicit_bits() - + binary_format::minimum_exponent(); + answer.power2 = bigmant.bit_length() - 64 + bias; + + round(answer, [truncated](adjusted_mantissa &a, int32_t shift) { + round_nearest_tie_even( + a, shift, + [truncated](bool is_odd, bool is_halfway, bool is_above) -> bool { + return is_above || (is_halfway && truncated) || + (is_odd && is_halfway); + }); + }); + return answer; } -namespace { +// the scaling here is quite simple: we have, for the real digits `m * 10^e`, +// and for the theoretical digits `n * 2^f`. Since `e` is always negative, +// to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`. +// we then need to scale by `2^(f- e)`, and then the two significant digits +// are of the same magnitude. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp( + bigint &bigmant, adjusted_mantissa am, int32_t exponent) noexcept { + bigint &real_digits = bigmant; + int32_t real_exp = exponent; + + // get the value of `b`, rounded down, and get a bigint representation of b+h + adjusted_mantissa am_b = am; + // gcc7 buf: use a lambda to remove the noexcept qualifier bug with + // -Wnoexcept-type. + round(am_b, + [](adjusted_mantissa &a, int32_t shift) { round_down(a, shift); }); + T b; + to_float(false, am_b, b); + adjusted_mantissa theor = to_extended_halfway(b); + bigint theor_digits(theor.mantissa); + int32_t theor_exp = theor.power2; + + // scale real digits and theor digits to be same power. + int32_t pow2_exp = theor_exp - real_exp; + uint32_t pow5_exp = uint32_t(-real_exp); + if (pow5_exp != 0) { + SIMDJSON_FASTFLOAT_ASSERT(theor_digits.pow5(pow5_exp)); + } + if (pow2_exp > 0) { + SIMDJSON_FASTFLOAT_ASSERT(theor_digits.pow2(uint32_t(pow2_exp))); + } else if (pow2_exp < 0) { + SIMDJSON_FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp))); + } + + // compare digits, and use it to direct rounding + int ord = real_digits.compare(theor_digits); + adjusted_mantissa answer = am; + round(answer, [ord](adjusted_mantissa &a, int32_t shift) { + round_nearest_tie_even( + a, shift, [ord](bool is_odd, bool _, bool __) -> bool { + static_cast(_); // not needed, since we've done our comparison + static_cast(__); // not needed, since we've done our comparison + if (ord > 0) { + return true; + } else if (ord < 0) { + return false; + } else { + return is_odd; + } + }); + }); + + return answer; +} -// remove all final zeroes -inline void trim(decimal &h) { - while ((h.num_digits > 0) && (h.digits[h.num_digits - 1] == 0)) { - h.num_digits--; +// parse the significant digits as a big integer to unambiguously round +// the significant digits. here, we are trying to determine how to round +// an extended float representation close to `b+h`, halfway between `b` +// (the float rounded-down) and `b+u`, the next positive float. this +// algorithm is always correct, and uses one of two approaches. when +// the exponent is positive relative to the significant digits (such as +// 1234), we create a big-integer representation, get the high 64-bits, +// determine if any lower bits are truncated, and use that to direct +// rounding. in case of a negative exponent relative to the significant +// digits (such as 1.2345), we create a theoretical representation of +// `b` as a big-integer type, scaled to the same binary exponent as +// the actual digits. we then compare the big integer representations +// of both, and use that to direct rounding. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +digit_comp(parsed_number_string_t &num, adjusted_mantissa am) noexcept { + // remove the invalid exponent bias + am.power2 -= invalid_am_bias; + + int32_t sci_exp = + scientific_exponent(num.mantissa, static_cast(num.exponent)); + size_t max_digits = binary_format::max_digits(); + size_t digits = 0; + bigint bigmant; + parse_mantissa(bigmant, num, max_digits, digits); + // can't underflow, since digits is at most max_digits. + int32_t exponent = sci_exp + 1 - int32_t(digits); + if (exponent >= 0) { + return positive_digit_comp(bigmant, exponent); + } else { + return negative_digit_comp(bigmant, am, exponent); } } -uint32_t number_of_digits_decimal_left_shift(decimal &h, uint32_t shift) { - shift &= 63; - const static uint16_t number_of_digits_decimal_left_shift_table[65] = { - 0x0000, 0x0800, 0x0801, 0x0803, 0x1006, 0x1009, 0x100D, 0x1812, 0x1817, - 0x181D, 0x2024, 0x202B, 0x2033, 0x203C, 0x2846, 0x2850, 0x285B, 0x3067, - 0x3073, 0x3080, 0x388E, 0x389C, 0x38AB, 0x38BB, 0x40CC, 0x40DD, 0x40EF, - 0x4902, 0x4915, 0x4929, 0x513E, 0x5153, 0x5169, 0x5180, 0x5998, 0x59B0, - 0x59C9, 0x61E3, 0x61FD, 0x6218, 0x6A34, 0x6A50, 0x6A6D, 0x6A8B, 0x72AA, - 0x72C9, 0x72E9, 0x7B0A, 0x7B2B, 0x7B4D, 0x8370, 0x8393, 0x83B7, 0x83DC, - 0x8C02, 0x8C28, 0x8C4F, 0x9477, 0x949F, 0x94C8, 0x9CF2, 0x051C, 0x051C, - 0x051C, 0x051C, - }; - uint32_t x_a = number_of_digits_decimal_left_shift_table[shift]; - uint32_t x_b = number_of_digits_decimal_left_shift_table[shift + 1]; - uint32_t num_new_digits = x_a >> 11; - uint32_t pow5_a = 0x7FF & x_a; - uint32_t pow5_b = 0x7FF & x_b; - const static uint8_t - number_of_digits_decimal_left_shift_table_powers_of_5[0x051C] = { - 5, 2, 5, 1, 2, 5, 6, 2, 5, 3, 1, 2, 5, 1, 5, 6, 2, 5, 7, 8, 1, 2, 5, - 3, 9, 0, 6, 2, 5, 1, 9, 5, 3, 1, 2, 5, 9, 7, 6, 5, 6, 2, 5, 4, 8, 8, - 2, 8, 1, 2, 5, 2, 4, 4, 1, 4, 0, 6, 2, 5, 1, 2, 2, 0, 7, 0, 3, 1, 2, - 5, 6, 1, 0, 3, 5, 1, 5, 6, 2, 5, 3, 0, 5, 1, 7, 5, 7, 8, 1, 2, 5, 1, - 5, 2, 5, 8, 7, 8, 9, 0, 6, 2, 5, 7, 6, 2, 9, 3, 9, 4, 5, 3, 1, 2, 5, - 3, 8, 1, 4, 6, 9, 7, 2, 6, 5, 6, 2, 5, 1, 9, 0, 7, 3, 4, 8, 6, 3, 2, - 8, 1, 2, 5, 9, 5, 3, 6, 7, 4, 3, 1, 6, 4, 0, 6, 2, 5, 4, 7, 6, 8, 3, - 7, 1, 5, 8, 2, 0, 3, 1, 2, 5, 2, 3, 8, 4, 1, 8, 5, 7, 9, 1, 0, 1, 5, - 6, 2, 5, 1, 1, 9, 2, 0, 9, 2, 8, 9, 5, 5, 0, 7, 8, 1, 2, 5, 5, 9, 6, - 0, 4, 6, 4, 4, 7, 7, 5, 3, 9, 0, 6, 2, 5, 2, 9, 8, 0, 2, 3, 2, 2, 3, - 8, 7, 6, 9, 5, 3, 1, 2, 5, 1, 4, 9, 0, 1, 1, 6, 1, 1, 9, 3, 8, 4, 7, - 6, 5, 6, 2, 5, 7, 4, 5, 0, 5, 8, 0, 5, 9, 6, 9, 2, 3, 8, 2, 8, 1, 2, - 5, 3, 7, 2, 5, 2, 9, 0, 2, 9, 8, 4, 6, 1, 9, 1, 4, 0, 6, 2, 5, 1, 8, - 6, 2, 6, 4, 5, 1, 4, 9, 2, 3, 0, 9, 5, 7, 0, 3, 1, 2, 5, 9, 3, 1, 3, - 2, 2, 5, 7, 4, 6, 1, 5, 4, 7, 8, 5, 1, 5, 6, 2, 5, 4, 6, 5, 6, 6, 1, - 2, 8, 7, 3, 0, 7, 7, 3, 9, 2, 5, 7, 8, 1, 2, 5, 2, 3, 2, 8, 3, 0, 6, - 4, 3, 6, 5, 3, 8, 6, 9, 6, 2, 8, 9, 0, 6, 2, 5, 1, 1, 6, 4, 1, 5, 3, - 2, 1, 8, 2, 6, 9, 3, 4, 8, 1, 4, 4, 5, 3, 1, 2, 5, 5, 8, 2, 0, 7, 6, - 6, 0, 9, 1, 3, 4, 6, 7, 4, 0, 7, 2, 2, 6, 5, 6, 2, 5, 2, 9, 1, 0, 3, - 8, 3, 0, 4, 5, 6, 7, 3, 3, 7, 0, 3, 6, 1, 3, 2, 8, 1, 2, 5, 1, 4, 5, - 5, 1, 9, 1, 5, 2, 2, 8, 3, 6, 6, 8, 5, 1, 8, 0, 6, 6, 4, 0, 6, 2, 5, - 7, 2, 7, 5, 9, 5, 7, 6, 1, 4, 1, 8, 3, 4, 2, 5, 9, 0, 3, 3, 2, 0, 3, - 1, 2, 5, 3, 6, 3, 7, 9, 7, 8, 8, 0, 7, 0, 9, 1, 7, 1, 2, 9, 5, 1, 6, - 6, 0, 1, 5, 6, 2, 5, 1, 8, 1, 8, 9, 8, 9, 4, 0, 3, 5, 4, 5, 8, 5, 6, - 4, 7, 5, 8, 3, 0, 0, 7, 8, 1, 2, 5, 9, 0, 9, 4, 9, 4, 7, 0, 1, 7, 7, - 2, 9, 2, 8, 2, 3, 7, 9, 1, 5, 0, 3, 9, 0, 6, 2, 5, 4, 5, 4, 7, 4, 7, - 3, 5, 0, 8, 8, 6, 4, 6, 4, 1, 1, 8, 9, 5, 7, 5, 1, 9, 5, 3, 1, 2, 5, - 2, 2, 7, 3, 7, 3, 6, 7, 5, 4, 4, 3, 2, 3, 2, 0, 5, 9, 4, 7, 8, 7, 5, - 9, 7, 6, 5, 6, 2, 5, 1, 1, 3, 6, 8, 6, 8, 3, 7, 7, 2, 1, 6, 1, 6, 0, - 2, 9, 7, 3, 9, 3, 7, 9, 8, 8, 2, 8, 1, 2, 5, 5, 6, 8, 4, 3, 4, 1, 8, - 8, 6, 0, 8, 0, 8, 0, 1, 4, 8, 6, 9, 6, 8, 9, 9, 4, 1, 4, 0, 6, 2, 5, - 2, 8, 4, 2, 1, 7, 0, 9, 4, 3, 0, 4, 0, 4, 0, 0, 7, 4, 3, 4, 8, 4, 4, - 9, 7, 0, 7, 0, 3, 1, 2, 5, 1, 4, 2, 1, 0, 8, 5, 4, 7, 1, 5, 2, 0, 2, - 0, 0, 3, 7, 1, 7, 4, 2, 2, 4, 8, 5, 3, 5, 1, 5, 6, 2, 5, 7, 1, 0, 5, - 4, 2, 7, 3, 5, 7, 6, 0, 1, 0, 0, 1, 8, 5, 8, 7, 1, 1, 2, 4, 2, 6, 7, - 5, 7, 8, 1, 2, 5, 3, 5, 5, 2, 7, 1, 3, 6, 7, 8, 8, 0, 0, 5, 0, 0, 9, - 2, 9, 3, 5, 5, 6, 2, 1, 3, 3, 7, 8, 9, 0, 6, 2, 5, 1, 7, 7, 6, 3, 5, - 6, 8, 3, 9, 4, 0, 0, 2, 5, 0, 4, 6, 4, 6, 7, 7, 8, 1, 0, 6, 6, 8, 9, - 4, 5, 3, 1, 2, 5, 8, 8, 8, 1, 7, 8, 4, 1, 9, 7, 0, 0, 1, 2, 5, 2, 3, - 2, 3, 3, 8, 9, 0, 5, 3, 3, 4, 4, 7, 2, 6, 5, 6, 2, 5, 4, 4, 4, 0, 8, - 9, 2, 0, 9, 8, 5, 0, 0, 6, 2, 6, 1, 6, 1, 6, 9, 4, 5, 2, 6, 6, 7, 2, - 3, 6, 3, 2, 8, 1, 2, 5, 2, 2, 2, 0, 4, 4, 6, 0, 4, 9, 2, 5, 0, 3, 1, - 3, 0, 8, 0, 8, 4, 7, 2, 6, 3, 3, 3, 6, 1, 8, 1, 6, 4, 0, 6, 2, 5, 1, - 1, 1, 0, 2, 2, 3, 0, 2, 4, 6, 2, 5, 1, 5, 6, 5, 4, 0, 4, 2, 3, 6, 3, - 1, 6, 6, 8, 0, 9, 0, 8, 2, 0, 3, 1, 2, 5, 5, 5, 5, 1, 1, 1, 5, 1, 2, - 3, 1, 2, 5, 7, 8, 2, 7, 0, 2, 1, 1, 8, 1, 5, 8, 3, 4, 0, 4, 5, 4, 1, - 0, 1, 5, 6, 2, 5, 2, 7, 7, 5, 5, 5, 7, 5, 6, 1, 5, 6, 2, 8, 9, 1, 3, - 5, 1, 0, 5, 9, 0, 7, 9, 1, 7, 0, 2, 2, 7, 0, 5, 0, 7, 8, 1, 2, 5, 1, - 3, 8, 7, 7, 7, 8, 7, 8, 0, 7, 8, 1, 4, 4, 5, 6, 7, 5, 5, 2, 9, 5, 3, - 9, 5, 8, 5, 1, 1, 3, 5, 2, 5, 3, 9, 0, 6, 2, 5, 6, 9, 3, 8, 8, 9, 3, - 9, 0, 3, 9, 0, 7, 2, 2, 8, 3, 7, 7, 6, 4, 7, 6, 9, 7, 9, 2, 5, 5, 6, - 7, 6, 2, 6, 9, 5, 3, 1, 2, 5, 3, 4, 6, 9, 4, 4, 6, 9, 5, 1, 9, 5, 3, - 6, 1, 4, 1, 8, 8, 8, 2, 3, 8, 4, 8, 9, 6, 2, 7, 8, 3, 8, 1, 3, 4, 7, - 6, 5, 6, 2, 5, 1, 7, 3, 4, 7, 2, 3, 4, 7, 5, 9, 7, 6, 8, 0, 7, 0, 9, - 4, 4, 1, 1, 9, 2, 4, 4, 8, 1, 3, 9, 1, 9, 0, 6, 7, 3, 8, 2, 8, 1, 2, - 5, 8, 6, 7, 3, 6, 1, 7, 3, 7, 9, 8, 8, 4, 0, 3, 5, 4, 7, 2, 0, 5, 9, - 6, 2, 2, 4, 0, 6, 9, 5, 9, 5, 3, 3, 6, 9, 1, 4, 0, 6, 2, 5, - }; - const uint8_t *pow5 = - &number_of_digits_decimal_left_shift_table_powers_of_5[pow5_a]; - uint32_t i = 0; - uint32_t n = pow5_b - pow5_a; - for (; i < n; i++) { - if (i >= h.num_digits) { - return num_new_digits - 1; - } else if (h.digits[i] == pow5[i]) { - continue; - } else if (h.digits[i] < pow5[i]) { - return num_new_digits - 1; - } else { - return num_new_digits; +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_PARSE_NUMBER_H +#define SIMDJSON_FASTFLOAT_PARSE_NUMBER_H + + +#include +#include +#include +#include + +namespace simdjson_fast_float { + +namespace detail { +/** + * Special case +inf, -inf, nan, infinity, -infinity. + * The case comparisons could be made much faster given that we know that the + * strings a null-free and fixed. + **/ +template +from_chars_result_t + SIMDJSON_FASTFLOAT_CONSTEXPR14 parse_infnan(UC const *first, UC const *last, + T &value, chars_format fmt) noexcept { + from_chars_result_t answer{}; + answer.ptr = first; + answer.ec = std::errc(); // be optimistic + // assume first < last, so dereference without checks; + bool const minusSign = (*first == UC('-')); + // C++17 20.19.3.(7.1) explicitly forbids '+' sign here + if ((*first == UC('-')) || + (uint64_t(fmt & chars_format::allow_leading_plus) && + (*first == UC('+')))) { + ++first; + } + if (last - first >= 3) { + if (simdjson_fastfloat_strncasecmp3(first, str_const_nan())) { + answer.ptr = (first += 3); + value = minusSign ? -std::numeric_limits::quiet_NaN() + : std::numeric_limits::quiet_NaN(); + // Check for possible nan(n-char-seq-opt), C++17 20.19.3.7, + // C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan). + if (first != last && *first == UC('(')) { + for (UC const *ptr = first + 1; ptr != last; ++ptr) { + if (*ptr == UC(')')) { + answer.ptr = ptr + 1; // valid nan(n-char-seq-opt) + break; + } else if (!((UC('a') <= *ptr && *ptr <= UC('z')) || + (UC('A') <= *ptr && *ptr <= UC('Z')) || + (UC('0') <= *ptr && *ptr <= UC('9')) || *ptr == UC('_'))) + break; // forbidden char, not nan(n-char-seq-opt) + } + } + return answer; + } + if (simdjson_fastfloat_strncasecmp3(first, str_const_inf())) { + if ((last - first >= 8) && + simdjson_fastfloat_strncasecmp5(first + 3, str_const_inf() + 3)) { + answer.ptr = first + 8; + } else { + answer.ptr = first + 3; + } + value = minusSign ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + return answer; } } - return num_new_digits; + answer.ec = std::errc::invalid_argument; + return answer; +} + +/** + * Returns true if the floating-pointing rounding mode is to 'nearest'. + * It is the default on most system. This function is meant to be inexpensive. + * Credit : @mwalcott3 + */ +simdjson_fastfloat_really_inline bool rounds_to_nearest() noexcept { + // https://lemire.me/blog/2020/06/26/gcc-not-nearest/ +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + return false; +#endif + // See + // A fast function to check your floating-point rounding mode + // https://lemire.me/blog/2022/11/16/a-fast-function-to-check-your-floating-point-rounding-mode/ + // + // This function is meant to be equivalent to : + // prior: #include + // return fegetround() == FE_TONEAREST; + // However, it is expected to be much faster than the fegetround() + // function call. + // + // The volatile keyword prevents the compiler from computing the function + // at compile-time. + // There might be other ways to prevent compile-time optimizations (e.g., + // asm). The value does not need to be std::numeric_limits::min(), any + // small value so that 1 + x should round to 1 would do (after accounting for + // excess precision, as in 387 instructions). + static float volatile fmin = (std::numeric_limits::min)(); + float fmini = fmin; // we copy it so that it gets loaded at most once. +// +// Explanation: +// Only when fegetround() == FE_TONEAREST do we have that +// fmin + 1.0f == 1.0f - fmin. +// +// FE_UPWARD: +// fmin + 1.0f > 1 +// 1.0f - fmin == 1 +// +// FE_DOWNWARD or FE_TOWARDZERO: +// fmin + 1.0f == 1 +// 1.0f - fmin < 1 +// +// Note: This may fail to be accurate if fast-math has been +// enabled, as rounding conventions may not apply. +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(push) +// todo: is there a VS warning? +// see +// https://stackoverflow.com/questions/46079446/is-there-a-warning-for-floating-point-equality-checking-in-visual-studio-2013 +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wfloat-equal" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + return (fmini + 1.0f == 1.0f - fmini); +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(pop) +#elif defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif } -} // end of anonymous namespace +} // namespace detail -uint64_t round(decimal &h) { - if ((h.num_digits == 0) || (h.decimal_point < 0)) { - return 0; - } else if (h.decimal_point > 18) { - return UINT64_MAX; - } - // at this point, we know that h.decimal_point >= 0 - uint32_t dp = uint32_t(h.decimal_point); - uint64_t n = 0; - for (uint32_t i = 0; i < dp; i++) { - n = (10 * n) + ((i < h.num_digits) ? h.digits[i] : 0); +template struct from_chars_caller { + template + SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_advanced(first, last, value, options); } - bool round_up = false; - if (dp < h.num_digits) { - round_up = h.digits[dp] >= 5; // normally, we round up - // but we may need to round to even! - if ((h.digits[dp] == 5) && (dp + 1 == h.num_digits)) { - round_up = h.truncated || ((dp > 0) && (1 & h.digits[dp - 1])); - } +}; + +#ifdef __STDCPP_FLOAT32_T__ +template <> struct from_chars_caller { + template + SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, std::float32_t &value, + parse_options_t options) noexcept { + // if std::float32_t is defined, and we are in C++23 mode; macro set for + // float32; set value to float due to equivalence between float and + // float32_t + float val = 0.0f; + auto ret = from_chars_advanced(first, last, val, options); + value = val; + return ret; } - if (round_up) { - n++; +}; +#endif + +#ifdef __STDCPP_FLOAT64_T__ +template <> struct from_chars_caller { + template + SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, std::float64_t &value, + parse_options_t options) noexcept { + // if std::float64_t is defined, and we are in C++23 mode; macro set for + // float64; set value as double due to equivalence between double and + // float64_t + double val = 0.0; + auto ret = from_chars_advanced(first, last, val, options); + value = val; + return ret; } - return n; +}; +#endif + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, + chars_format fmt /*= chars_format::general*/) noexcept { + return from_chars_caller::call(first, last, value, + parse_options_t(fmt)); } -// computes h * 2^-shift -void decimal_left_shift(decimal &h, uint32_t shift) { - if (h.num_digits == 0) { - return; - } - uint32_t num_new_digits = number_of_digits_decimal_left_shift(h, shift); - int32_t read_index = int32_t(h.num_digits - 1); - uint32_t write_index = h.num_digits - 1 + num_new_digits; - uint64_t n = 0; - - while (read_index >= 0) { - n += uint64_t(h.digits[read_index]) << shift; - uint64_t quotient = n / 10; - uint64_t remainder = n - (10 * quotient); - if (write_index < max_digits) { - h.digits[write_index] = uint8_t(remainder); - } else if (remainder > 0) { - h.truncated = true; - } - n = quotient; - write_index--; - read_index--; - } - while (n > 0) { - uint64_t quotient = n / 10; - uint64_t remainder = n - (10 * quotient); - if (write_index < max_digits) { - h.digits[write_index] = uint8_t(remainder); - } else if (remainder > 0) { - h.truncated = true; - } - n = quotient; - write_index--; - } - h.num_digits += num_new_digits; - if (h.num_digits > max_digits) { - h.num_digits = max_digits; - } - h.decimal_point += int32_t(num_new_digits); - trim(h); -} - -// computes h * 2^shift -void decimal_right_shift(decimal &h, uint32_t shift) { - uint32_t read_index = 0; - uint32_t write_index = 0; - - uint64_t n = 0; - - while ((n >> shift) == 0) { - if (read_index < h.num_digits) { - n = (10 * n) + h.digits[read_index++]; - } else if (n == 0) { - return; +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +clinger_fast_path_impl(uint64_t mantissa, int64_t exponent, bool is_negative, + T &value) noexcept { + // The implementation of the Clinger's fast path is convoluted because + // we want round-to-nearest in all cases, irrespective of the rounding mode + // selected on the thread. + // We proceed optimistically, assuming that detail::rounds_to_nearest() + // returns true. + if (binary_format::min_exponent_fast_path() <= exponent && + exponent <= binary_format::max_exponent_fast_path() && + mantissa <= binary_format::max_mantissa_fast_path()) { + // The mantissa bound above is a necessary condition for BOTH branches + // below: the rounding-mode-dependent branch checks the tighter + // max_mantissa_fast_path(exponent) <= max_mantissa_fast_path(). Testing + // it before detail::rounds_to_nearest() spares long-mantissa inputs + // (which can never take the fast path) the volatile-float probe. + // + // Unfortunately, the conventional Clinger's fast path is only possible + // when the system rounds to the nearest float. + // + // We expect the next branch to almost always be selected. + // We could check it first (before the previous branch), but + // there might be performance advantages at having the check + // be last. + if (!cpp20_and_in_constexpr() && detail::rounds_to_nearest()) { + // We have that fegetround() == FE_TONEAREST. + // Next is Clinger's fast path. + value = T(mantissa); + if (exponent < 0) { + value = value / binary_format::exact_power_of_ten(-exponent); + } else { + value = value * binary_format::exact_power_of_ten(exponent); + } + if (is_negative) { + value = -value; + } + return true; } else { - while ((n >> shift) == 0) { - n = 10 * n; - read_index++; + // We do not have that fegetround() == FE_TONEAREST. + // Next is a modified Clinger's fast path, inspired by Jakub Jelinek's + // proposal + if (exponent >= 0 && + mantissa <= binary_format::max_mantissa_fast_path(exponent)) { +#if defined(__clang__) || defined(SIMDJSON_FASTFLOAT_32BIT) + // Clang may map 0 to -0.0 when fegetround() == FE_DOWNWARD + if (mantissa == 0) { + value = is_negative ? T(-0.) : T(0.); + return true; + } +#endif + value = T(mantissa) * binary_format::exact_power_of_ten(exponent); + if (is_negative) { + value = -value; + } + return true; } - break; } } - h.decimal_point -= int32_t(read_index - 1); - if (h.decimal_point < -decimal_point_range) { // it is zero - h.num_digits = 0; - h.decimal_point = 0; - h.negative = false; - h.truncated = false; - return; + return false; +} + +/** + * This function overload takes parsed_number_string_t structure that is created + * and populated either by from_chars_advanced function taking chars range and + * parsing options or other parsing custom function implemented by user. + */ +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_advanced(parsed_number_string_t &pns, T &value) noexcept { + static_assert(is_supported_float_type::value, + "only some floating-point types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + from_chars_result_t answer; + + answer.ec = std::errc(); // be optimistic + answer.ptr = pns.lastmatch; + + if (!pns.too_many_digits && + clinger_fast_path_impl(pns.mantissa, pns.exponent, pns.negative, value)) + return answer; + + adjusted_mantissa am = + compute_float>(pns.exponent, pns.mantissa); + if (pns.too_many_digits && am.power2 >= 0) { + if (am != compute_float>(pns.exponent, pns.mantissa + 1)) { + am = compute_error>(pns.exponent, pns.mantissa); + } } - uint64_t mask = (uint64_t(1) << shift) - 1; - while (read_index < h.num_digits) { - uint8_t new_digit = uint8_t(n >> shift); - n = (10 * (n & mask)) + h.digits[read_index++]; - h.digits[write_index++] = new_digit; + // If we called compute_float>(pns.exponent, pns.mantissa) + // and we have an invalid power (am.power2 < 0), then we need to go the long + // way around again. This is very uncommon. + if (am.power2 < 0) { + am = digit_comp(pns, am); } - while (n > 0) { - uint8_t new_digit = uint8_t(n >> shift); - n = 10 * (n & mask); - if (write_index < max_digits) { - h.digits[write_index++] = new_digit; - } else if (new_digit > 0) { - h.truncated = true; - } + to_float(pns.negative, am, value); + // Test for over/underflow. + if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || + am.power2 == binary_format::infinite_power()) { + answer.ec = std::errc::result_out_of_range; } - h.num_digits = write_index; - trim(h); + return answer; } -template adjusted_mantissa compute_float(decimal &d) { - adjusted_mantissa answer; - if (d.num_digits == 0) { - // should be zero - answer.power2 = 0; - answer.mantissa = 0; - return answer; - } - // At this point, going further, we can assume that d.num_digits > 0. - // We want to guard against excessive decimal point values because - // they can result in long running times. Indeed, we do - // shifts by at most 60 bits. We have that log(10**400)/log(2**60) ~= 22 - // which is fine, but log(10**299995)/log(2**60) ~= 16609 which is not - // fine (runs for a long time). - // - if(d.decimal_point < -324) { - // We have something smaller than 1e-324 which is always zero - // in binary64 and binary32. - // It should be zero. - answer.power2 = 0; - answer.mantissa = 0; - return answer; - } else if(d.decimal_point >= 310) { - // We have something at least as large as 0.1e310 which is - // always infinite. - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; +// Slow path: re-parse materializing the integer/fraction spans the hot no-span +// parse skipped, then run the full algorithm. The two callers reach it only +// through a simdjson_fastfloat_unlikely branch, so the optimizer keeps this re-parse off +// the hot path on its own (no function-level noinline needed). +// from_chars_advanced already handles both the too_many_digits disambiguation +// and the am.power2<0 digit_comp recompute, so both slow branches collapse to +// one helper call. +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +parse_number_slow_path(UC const *first, UC const *last, T &value, + parse_options_t options, bool bjf) noexcept { + parsed_number_string_t pns = + bjf ? parse_number_string(first, last, options, true) + : parse_number_string(first, last, options, true); + return from_chars_advanced(pns, value); +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_float_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + + static_assert(is_supported_float_type::value, + "only some floating-point types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + + from_chars_result_t answer; + if (uint64_t(fmt & chars_format::skip_white_space)) { + while ((first != last) && simdjson_fast_float::is_space(*first)) { + first++; + } + } + if (first == last) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; return answer; } - - static const uint32_t max_shift = 60; - static const uint32_t num_powers = 19; - static const uint8_t powers[19] = { - 0, 3, 6, 9, 13, 16, 19, 23, 26, 29, // - 33, 36, 39, 43, 46, 49, 53, 56, 59, // - }; - int32_t exp2 = 0; - while (d.decimal_point > 0) { - uint32_t n = uint32_t(d.decimal_point); - uint32_t shift = (n < num_powers) ? powers[n] : max_shift; - decimal_right_shift(d, shift); - if (d.decimal_point < -decimal_point_range) { - // should be zero - answer.power2 = 0; - answer.mantissa = 0; + bool const bjf = uint64_t(fmt & detail::basic_json_fmt) != 0; + + // Fast path: parse WITHOUT materializing the integer/fraction spans (read + // only by the rare slow paths). Skipping their stores keeps the fat + // parsed_number_string_t off the hot path. store_spans is a runtime argument, + // so this reuses the single parse_number_string instantiation. + parsed_number_string_t pns = + bjf ? parse_number_string(first, last, options, false) + : parse_number_string(first, last, options, false); + if (!pns.valid) { + if (uint64_t(fmt & chars_format::no_infnan)) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; return answer; - } - exp2 += int32_t(shift); - } - // We shift left toward [1/2 ... 1]. - while (d.decimal_point <= 0) { - uint32_t shift; - if (d.decimal_point == 0) { - if (d.digits[0] >= 5) { - break; - } - shift = (d.digits[0] < 2) ? 2 : 1; } else { - uint32_t n = uint32_t(-d.decimal_point); - shift = (n < num_powers) ? powers[n] : max_shift; + return detail::parse_infnan(first, last, value, fmt); } - decimal_left_shift(d, shift); - if (d.decimal_point > decimal_point_range) { - // we want to get infinity: - answer.power2 = 0xFF; - answer.mantissa = 0; - return answer; - } - exp2 -= int32_t(shift); } - // We are now in the range [1/2 ... 1] but the binary format uses [1 ... 2]. - exp2--; - constexpr int32_t minimum_exponent = binary::minimum_exponent(); - while ((minimum_exponent + 1) > exp2) { - uint32_t n = uint32_t((minimum_exponent + 1) - exp2); - if (n > max_shift) { - n = max_shift; - } - decimal_right_shift(d, n); - exp2 += int32_t(n); + + // Slow path A (rare): > 19 significant digits. The no-span parse left the + // mantissa un-truncated and skipped the span-based recompute; the cold helper + // re-parses with spans and runs the full algorithm. + // +// We have to disable -Wc++20-extensions for the [[unlikely]] attribute +// See comment for @jwakely at +// https://github.com/fastfloat/simdjson_fast_float/pull/387#discussion_r3366943539 +// This is unfortunate. +#ifdef __clang__ +#pragma clang diagnostic push +#if (!defined(__APPLE_CC__) && __clang_major__ >= 10) || (__clang_major__ >= 13) +#pragma clang diagnostic ignored "-Wc++20-extensions" +#endif +#endif + if simdjson_fastfloat_unlikely (pns.too_many_digits) { + return parse_number_slow_path(first, last, value, options, bjf); } - if ((exp2 - minimum_exponent) >= binary::infinite_power()) { - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; + answer.ec = std::errc(); // be optimistic + answer.ptr = pns.lastmatch; + + if (clinger_fast_path_impl(pns.mantissa, pns.exponent, pns.negative, value)) { return answer; } - const int mantissa_size_in_bits = binary::mantissa_explicit_bits() + 1; - decimal_left_shift(d, mantissa_size_in_bits); - - uint64_t mantissa = round(d); - // It is possible that we have an overflow, in which case we need - // to shift back. - if (mantissa >= (uint64_t(1) << mantissa_size_in_bits)) { - decimal_right_shift(d, 1); - exp2 += 1; - mantissa = round(d); - if ((exp2 - minimum_exponent) >= binary::infinite_power()) { - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } + adjusted_mantissa am = + compute_float>(pns.exponent, pns.mantissa); + // Slow path B (rare): Eisel-Lemire could not resolve; digit_comp needs the + // integer/fraction spans. Route to the cold helper (clinger there is a + // dead-effect since it already failed here; the cold re-parse + digit_comp + // via from_chars_advanced reproduces this branch). + if simdjson_fastfloat_unlikely (am.power2 < 0) { + return parse_number_slow_path(first, last, value, options, bjf); } - answer.power2 = exp2 - binary::minimum_exponent(); - if (mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) { - answer.power2--; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + to_float(pns.negative, am, value); + // Test for over/underflow. + if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || + am.power2 == binary_format::infinite_power()) { + answer.ec = std::errc::result_out_of_range; } - answer.mantissa = - mantissa & ((uint64_t(1) << binary::mantissa_explicit_bits()) - 1); return answer; } -template -adjusted_mantissa parse_long_mantissa(const char *first) { - decimal d = parse_decimal(first); - return compute_float(d); +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, int base) noexcept { + + static_assert(is_supported_integer_type::value, + "only integer types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + parse_options_t options; + options.base = base; + return from_chars_advanced(first, last, value, options); } -template -adjusted_mantissa parse_long_mantissa(const char *first, const char *end) { - decimal d = parse_decimal(first, end); - return compute_float(d); +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept { + T value; + if (clinger_fast_path_impl(mantissa, decimal_exponent, false, value)) + return value; + + adjusted_mantissa am = + compute_float>(decimal_exponent, mantissa); + to_float(false, am, value); + return value; } -double from_chars(const char *first) noexcept { - bool negative = first[0] == '-'; - if (negative) { - first++; - } - adjusted_mantissa am = parse_long_mantissa>(first); - uint64_t word = am.mantissa; - word |= uint64_t(am.power2) - << binary_format::mantissa_explicit_bits(); - word = negative ? word | (uint64_t(1) << binary_format::sign_index()) - : word; - double value; - std::memcpy(&value, &word, sizeof(double)); +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept { + const bool is_negative = mantissa < 0; + const uint64_t m = static_cast(is_negative ? -mantissa : mantissa); + + T value; + if (clinger_fast_path_impl(m, decimal_exponent, is_negative, value)) + return value; + + adjusted_mantissa am = compute_float>(decimal_exponent, m); + to_float(is_negative, am, value); return value; } +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(mantissa, decimal_exponent); +} -double from_chars(const char *first, const char *end) noexcept { - bool negative = first[0] == '-'; - if (negative) { - first++; +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(mantissa, decimal_exponent); +} + +// the following overloads are here to avoid surprising ambiguity for int, +// unsigned, etc. +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value && + std::is_integral::value && + !std::is_signed::value, + T>::type + integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), + decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value && + std::is_integral::value && + std::is_signed::value, + T>::type + integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), + decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 typename std::enable_if< + std::is_integral::value && !std::is_signed::value, double>::type +integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 typename std::enable_if< + std::is_integral::value && std::is_signed::value, double>::type +integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_int_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + + static_assert(is_supported_integer_type::value, + "only integer types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + int const base = options.base; + + from_chars_result_t answer; + if (uint64_t(fmt & chars_format::skip_white_space)) { + while ((first != last) && simdjson_fast_float::is_space(*first)) { + first++; + } + } + if (first == last || base < 2 || base > 36) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + + return parse_int_string(first, last, value, options); +} + +template struct from_chars_advanced_caller { + static_assert(TypeIx > 0, "unsupported type"); +}; + +template <> struct from_chars_advanced_caller<1> { + template + simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_float_advanced(first, last, value, options); + } +}; + +template <> struct from_chars_advanced_caller<2> { + template + simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_int_advanced(first, last, value, options); + } +}; + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_advanced_caller< + size_t(is_supported_float_type::value) + + 2 * size_t(is_supported_integer_type::value)>::call(first, last, value, + options); +} + +} // namespace simdjson_fast_float + +#endif + +/* end file simdjson/internal/fast_float.h */ + +#include +#include +#include + +namespace simdjson { +namespace internal { + +/** + * These functions handle floating-point parsing when the fast path in + * numberparsing.h gives up: more than 19 digits in the decimal mantissa, an + * exponent outside the range the power-of-five table covers, or one of the rare + * inputs where the truncated Eisel-Lemire product is not accurate enough to + * round. That should only be seen in adversarial scenarios; we do not expect + * production systems to even produce such floating-point numbers. + * + * The work is handed to fast_float (vendored in + * include/simdjson/internal/fast_float.h), which settles the rounding by + * comparing a bigint against a scaled power of five. It is correctly rounded, + * and quick enough that an adversarial document is no longer worth worrying + * about. + **/ + +namespace { + +// fast_float wants the end of the number, and the callers only promise that a +// number is followed by a character which cannot be part of one -- the input +// has already been validated against the JSON grammar, and the buffer is padded, +// so such a character is always there to be found. Locating it costs a pass over +// digits we are about to parse anyway, and in exchange fast_float can bound its +// inner loops instead of re-checking a far-away end pointer. +const char *find_end_of_number(const char *first) noexcept { + const char *p = first; + while ((*p >= '0' && *p <= '9') || *p == '-' || *p == '+' || *p == '.' || + *p == 'e' || *p == 'E') { + p++; } - adjusted_mantissa am = parse_long_mantissa>(first, end); - uint64_t word = am.mantissa; - word |= uint64_t(am.power2) - << binary_format::mantissa_explicit_bits(); - word = negative ? word | (uint64_t(1) << binary_format::sign_index()) - : word; - double value; - std::memcpy(&value, &word, sizeof(double)); + return p; +} + +// The input is JSON, so parse it under the JSON grammar: no hexadecimal, no +// leading plus, and no infinity or NaN spellings. Those are handled (or +// rejected) before we ever get here. +constexpr simdjson_fast_float::parse_options json_options{ + simdjson_fast_float::chars_format::json}; + +// fast_float reports result_out_of_range for a value at either edge of the +// format, writing +/-0 when it underflows and +/-infinity when it overflows. +// Both are exactly what the callers of these functions expect to receive: they +// accept a zero and treat an infinity as an error. A malformed number cannot +// happen on validated input, but if it somehow did, returning zero matches what +// the previous implementation did with digits it could not use. +template T parse_with_fast_float(const char *first, const char *end) noexcept { + T value{}; + auto answer = + simdjson_fast_float::from_chars_advanced(first, end, value, json_options); + if (answer.ec == std::errc::invalid_argument) { return T(0); } return value; } +} // namespace + +double from_chars(const char *first) noexcept { + return parse_with_fast_float(first, find_end_of_number(first)); +} + +double from_chars(const char *first, const char *end) noexcept { + return parse_with_fast_float(first, end); +} + +float from_chars_float(const char *first) noexcept { + return parse_with_fast_float(first, find_end_of_number(first)); +} + } // internal } // simdjson @@ -5109,7 +10245,8 @@ namespace internal { { SCALAR_DOCUMENT_AS_VALUE, "SCALAR_DOCUMENT_AS_VALUE: A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "}, { OUT_OF_BOUNDS, "OUT_OF_BOUNDS: Attempt to access location outside of document."}, { TRAILING_CONTENT, "TRAILING_CONTENT: Unexpected trailing content in the JSON input."}, - { OUT_OF_CAPACITY, "OUT_OF_CAPACITY: The capacity was exceeded, we cannot allocate enough memory."} + { OUT_OF_CAPACITY, "OUT_OF_CAPACITY: The capacity was exceeded, we cannot allocate enough memory."}, + { UNKNOWN_FIELD, "UNKNOWN_FIELD: The JSON object has a field that does not map to any member of the target type (deny_unknown_fields)."} }; // error_messages[] } // namespace internal @@ -7115,7 +12252,12 @@ class document; * 3) The stream_final mode allows us to truncate final * unterminated strings. It is useful in conjunction with streaming_partial. */ -enum class stage1_mode { regular, streaming_partial, streaming_final}; +enum class stage1_mode { + regular, + streaming_partial, streaming_final, + json_sequence_partial, json_sequence_final, + comma_delimited_partial, comma_delimited_final +}; /** * Returns true if mode == streaming_partial or mode == streaming_final @@ -7127,7 +12269,6 @@ inline bool is_streaming(stage1_mode mode) { // return (mode == stage1_mode::streaming_partial || mode == stage1_mode::streaming_final); } - namespace internal { @@ -7312,6 +12453,16 @@ class dom_parser_implementation { /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ bool _number_as_string{false}; + /** + * Whether the input buffer passed to parse() is *not* padded to len + + * SIMDJSON_PADDING bytes. When true, stage 2 string parsing avoids reading + * past buf+len (it finishes the final, near-the-end bytes from a small padded + * scratch buffer). This is set only by the no-padding DOM parse entry points + * (dom::parser::parse_unpadded); the default padded fast path leaves it false + * and is unaffected. + */ + bool _unpadded{false}; + protected: // Declaring these so that subclasses can use them to implement their constructors. @@ -7655,6 +12806,8 @@ enum instruction_set { LASX = 0x40000, //RVV = 0x80000, RVV_VLS = 0x100000, + SVE = 0x200000, + SVE2 = 0x400000, }; } // namespace internal @@ -7966,12 +13119,24 @@ POSSIBILITY OF SUCH DAMAGE. #include #if defined(_MSC_VER) #include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || defined(__FILC__) #include #endif #if defined(__loongarch__) && defined(__linux__) #include #endif +#if defined(__aarch64__) && defined(__linux__) + #include +#endif +#if (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) && defined(_WIN32) && !defined(_WINDOWS_) +// We avoid including (macro pollution); this matches the +// declaration in the Windows SDK (BOOL WINAPI IsProcessorFeaturePresent(DWORD)). +extern "C" __declspec(dllimport) int __stdcall IsProcessorFeaturePresent(unsigned long ProcessorFeature); +#endif + +#ifdef __FILC__ +#include +#endif namespace simdjson { namespace internal { @@ -7984,8 +13149,60 @@ static inline uint32_t detect_supported_architectures() { #elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +#if defined(__linux__) +// The kernel advertises SVE in AT_HWCAP and SVE2 in AT_HWCAP2. Older +// headers may not define these constants, so we provide the kernel's values +// (we deliberately do not include , which is not available on +// all toolchains, e.g., musl without linux-headers). +#ifndef AT_HWCAP2 +#define AT_HWCAP2 26 +#endif +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif // __linux__ + +#if defined(_WIN32) +// Only recent Windows SDKs define these processor features. +#ifndef PF_ARM_SVE_INSTRUCTIONS_AVAILABLE +#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46 +#endif +#ifndef PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE +#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47 +#endif +#endif // _WIN32 + static inline uint32_t detect_supported_architectures() { - return instruction_set::NEON; + // NEON is mandatory on AArch64. + uint32_t host_isa = instruction_set::NEON; +#if defined(__linux__) + unsigned long hwcap = getauxval(AT_HWCAP); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + if (hwcap & HWCAP_SVE) { + host_isa |= instruction_set::SVE; + // We only claim SVE2 when SVE is also present. Before Linux 6.14, the + // kernel set HWCAP2_SVE2 on processors implementing SME(2) but not SVE, + // because SVE2 instructions are available in streaming mode. Our SVE2 + // code runs in non-streaming mode and needs actual SVE. + if (hwcap2 & HWCAP2_SVE2) { + host_isa |= instruction_set::SVE2; + } + } +#elif defined(_WIN32) + if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) { + host_isa |= instruction_set::SVE; + // As on Linux, require SVE before claiming SVE2. + if (IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)) { + host_isa |= instruction_set::SVE2; + } + } +#endif + // On other systems (e.g., macOS, where Apple Silicon has no SVE), we only + // report NEON. + return host_isa; } #elif defined(__x86_64__) || defined(_M_AMD64) // x64 @@ -8023,7 +13240,7 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, *ebx = cpu_info[1]; *ecx = cpu_info[2]; *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || defined(__FILC__) uint32_t level = *eax; __get_cpuid(level, eax, ebx, ecx, edx); #else @@ -8040,6 +13257,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, static inline uint64_t xgetbv() { #if defined(_MSC_VER) return _xgetbv(0); +#elif defined(__FILC__) + return zxgetbv(); #else uint32_t xcr0_lo, xcr0_hi; asm volatile("xgetbv\n\t" : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); @@ -8601,7 +13820,7 @@ class implementation final : public simdjson::implementation { simdjson_inline implementation() : simdjson::implementation( "rvv_vls", "RISC-V V extension", - 0 + internal::instruction_set::RVV_VLS ) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, @@ -8976,7 +14195,14 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { /* result might be undefined when input_num is zero */ simdjson_inline int count_ones(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#else + // if the system supports SVE or CSSC, __builtin_popcountll + // might be compiled to fewer single instructions. For CSSC, + // __builtin_popcountll is compiled to a single instruction. + return __builtin_popcountll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO } @@ -9013,15 +14239,6 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace arm64 @@ -9285,6 +14502,7 @@ namespace { return vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); } + // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -9933,6 +15151,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -9944,6 +15165,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -9980,6 +15223,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace arm64 @@ -10070,7 +15378,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -10249,6 +15557,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -10288,6 +15597,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -10544,6 +15865,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -10581,6 +16115,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -10599,6 +16143,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -10615,26 +16181,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -10723,7 +16336,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -10806,15 +16419,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -10845,7 +16460,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -10894,7 +16523,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -10993,7 +16622,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11091,7 +16720,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11146,7 +16775,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11232,7 +16861,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11272,11 +16901,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -11287,9 +16925,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -11338,6 +16975,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -11490,11 +17218,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -11505,9 +17247,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -11556,6 +17297,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -11749,6 +17583,9 @@ class implementation final : public simdjson::implementation { #endif // SIMDJSON_ARM64_IMPLEMENTATION_H /* end file simdjson/arm64/implementation.h */ +// defining SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write +#define SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + /* including simdjson/arm64/begin.h: #include */ /* begin file simdjson/arm64/begin.h */ /* defining SIMDJSON_IMPLEMENTATION to "arm64" */ @@ -11861,7 +17698,14 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { /* result might be undefined when input_num is zero */ simdjson_inline int count_ones(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#else + // if the system supports SVE or CSSC, __builtin_popcountll + // might be compiled to fewer single instructions. For CSSC, + // __builtin_popcountll is compiled to a single instruction. + return __builtin_popcountll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO } @@ -11898,15 +17742,6 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace arm64 @@ -12170,6 +18005,7 @@ namespace { return vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); } + // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -13615,6 +19451,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace arm64 @@ -14048,7 +20174,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -14076,6 +20201,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -14158,7 +20352,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -14455,7 +20648,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -14472,6 +20665,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -14480,6 +20674,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -14528,7 +20723,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -14543,7 +20738,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -14555,8 +20750,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -14573,29 +20768,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -14623,16 +20818,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -14660,11 +20855,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -14704,7 +20917,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -14726,7 +20952,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -14912,9 +21151,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -14937,6 +21180,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -14991,73 +21305,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for arm64 */ -/* including generic/stage2/structural_iterator.h for arm64: #include */ -/* begin file generic/stage2/structural_iterator.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for arm64 */ /* including generic/stage2/tape_builder.h for arm64: #include */ /* begin file generic/stage2/tape_builder.h for arm64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -15080,12 +21327,8 @@ namespace arm64 { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -15138,88 +21381,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -15228,27 +21513,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -15256,7 +21562,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -15270,76 +21577,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -15352,13 +21725,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -15379,6 +21754,38 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { /* end file generic/stage2/tape_builder.h for arm64 */ /* end file generic/stage2/amalgamated.h for arm64 */ +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +namespace simdjson { namespace arm64 { namespace { namespace stage1 { + +// The generic bit_indexer::write emits the structural indexes in groups of +// four (up to 24), so a block with five set bits computes and stores eight +// indexes, three of them wasted. Typical JSON has four to eight structural +// characters per 64-byte block. This version writes the first four indexes +// unconditionally and then continues by groups of two, which cuts the wasted +// work on such blocks at the cost of one more branch for dense blocks. Both +// versions fall back to the same scalar loop past 24 indexes. +simdjson_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { + if (bits == 0) { return; } + + const int cnt = static_cast(count_ones(bits)); +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif + write_indexes<0, 4>(idx, bits); + if (simdjson_unlikely(4 < cnt)) { + write_indexes_stepped<4, 24, 2>(idx, bits, cnt); + } + if (simdjson_unlikely(24 < cnt)) { + for (int i = 24; i < cnt; ++i) { + write_index(idx, bits, i); + } + } + this->tail += cnt; +} + +}}}} // namespace simdjson::arm64::(anonymous)::stage1 + // // Stage 1 // @@ -15404,55 +21811,43 @@ namespace { using namespace simd; simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // Functional programming causes trouble with Visual Studio. - // Keeping this version in comments since it is much nicer: - // auto v = in.map([&](simd8 chunk) { - // auto nib_lo = chunk & 0xf; - // auto nib_hi = chunk.shr<4>(); - // auto shuf_lo = nib_lo.lookup_16(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - // auto shuf_hi = nib_hi.lookup_16(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - // return shuf_lo & shuf_hi; - // }); - const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - - simd8x64 v( - (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), - (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), - (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), - (in.chunks[3] & 0xf).lookup_16(table1) & (in.chunks[3].shr<4>()).lookup_16(table2) + const uint8x16_t op_table = simd8( + 0xff, 0, ',', ':', 0, '[', ']', '{', '}', 0, 0, 0, 0, 0, 0, 0 + ); + const uint8x16_t ws_table = simd8( + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0xff, 0, 0 ); + const uint8x16_t d0_0 = in.chunks[0]; + const uint8x16_t d0_1 = in.chunks[1]; + const uint8x16_t d0_2 = in.chunks[2]; + const uint8x16_t d0_3 = in.chunks[3]; - // We compute whitespace and op separately. If the code later only use one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). *However* if we only need spaces, - // it is likely that we will still compute 'v' above with two lookup_16: one - // could do it a bit cheaper. This is in contrast with the x64 implementations - // where we can, efficiently, do the white space and structural matching - // separately. One reason for this difference is that on ARM NEON, the table - // lookups either zero or leave unchanged the characters exceeding 0xF whereas - // on x64, the equivalent instruction (pshufb) automatically applies a mask, - // ignoring the 4 most significant bits. Thus the x64 implementation is - // optimized differently. This being said, if you use this code strictly - // just for minification (or just to identify the structural characters), - // there is a small untaken optimization opportunity here. We deliberately - // do not pick it up. + const uint8x16_t match_op_0 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_0, vdupq_n_u8(3)), 4)), d0_0); + const uint8x16_t match_op_1 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_1, vdupq_n_u8(3)), 4)), d0_1); + const uint8x16_t match_op_2 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_2, vdupq_n_u8(3)), 4)), d0_2); + const uint8x16_t match_op_3 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_3, vdupq_n_u8(3)), 4)), d0_3); - uint64_t op = simd8x64( - v.chunks[0].any_bits_set(0x7), - v.chunks[1].any_bits_set(0x7), - v.chunks[2].any_bits_set(0x7), - v.chunks[3].any_bits_set(0x7) - ).to_bitmask(); + const uint8x16_t match_ws_0 = vqtbx1q_u8(vceqq_u8(d0_0, vdupq_n_u8(' ')), ws_table, d0_0); + const uint8x16_t match_ws_1 = vqtbx1q_u8(vceqq_u8(d0_1, vdupq_n_u8(' ')), ws_table, d0_1); + const uint8x16_t match_ws_2 = vqtbx1q_u8(vceqq_u8(d0_2, vdupq_n_u8(' ')), ws_table, d0_2); + const uint8x16_t match_ws_3 = vqtbx1q_u8(vceqq_u8(d0_3, vdupq_n_u8(' ')), ws_table, d0_3); - uint64_t whitespace = simd8x64( - v.chunks[0].any_bits_set(0x18), - v.chunks[1].any_bits_set(0x18), - v.chunks[2].any_bits_set(0x18), - v.chunks[3].any_bits_set(0x18) - ).to_bitmask(); + const uint8x16_t bit_mask = simd8( + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 + ); + + uint8x16_t op_sum0 = vpaddq_u8(vandq_u8(match_op_0, bit_mask), vandq_u8(match_op_1, bit_mask)); + uint8x16_t ws_sum0 = vpaddq_u8(vandq_u8(match_ws_0, bit_mask), vandq_u8(match_ws_1, bit_mask)); + uint8x16_t op_sum1 = vpaddq_u8(vandq_u8(match_op_2, bit_mask), vandq_u8(match_op_3, bit_mask)); + uint8x16_t ws_sum1 = vpaddq_u8(vandq_u8(match_ws_2, bit_mask), vandq_u8(match_ws_3, bit_mask)); + op_sum0 = vpaddq_u8(op_sum0, op_sum1); + ws_sum0 = vpaddq_u8(ws_sum0, ws_sum1); + op_sum0 = vpaddq_u8(op_sum0, op_sum0); + ws_sum0 = vpaddq_u8(ws_sum0, ws_sum0); + const uint64_t op = vgetq_lane_u64(vreinterpretq_u64_u8(op_sum0), 0); + const uint64_t whitespace = vgetq_lane_u64(vreinterpretq_u64_u8(ws_sum0), 0); return { whitespace, op }; } @@ -15498,7 +21893,7 @@ simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_ return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); } -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { +simdjson_flatten simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { this->buf = _buf; this->len = _len; return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); @@ -15721,16 +22116,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace haswell @@ -16483,6 +22868,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -16494,6 +22882,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -16530,6 +22940,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace haswell @@ -16620,7 +23095,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -16799,6 +23274,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -16838,6 +23314,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -17094,6 +23582,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -17131,6 +23832,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -17149,6 +23860,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -17165,26 +23898,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -17273,7 +24053,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -17356,15 +24136,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -17395,7 +24177,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -17444,7 +24240,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -17543,7 +24339,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17641,7 +24437,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17696,7 +24492,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17782,7 +24578,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17822,11 +24618,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -17837,9 +24642,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -17888,6 +24692,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -18040,11 +24935,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -18055,9 +24964,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -18106,6 +25014,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -18465,16 +25466,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace haswell @@ -20024,6 +27015,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace haswell @@ -20457,7 +27738,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -20485,6 +27765,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -20567,7 +27916,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -20864,7 +28212,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -20881,6 +28229,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -20889,6 +28238,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -20937,7 +28287,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -20952,7 +28302,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -20964,8 +28314,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -20982,29 +28332,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -21032,16 +28382,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -21069,11 +28419,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -21113,7 +28481,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -21135,7 +28516,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -21321,9 +28715,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -21346,6 +28744,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -21400,73 +28869,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for haswell */ -/* including generic/stage2/structural_iterator.h for haswell: #include */ -/* begin file generic/stage2/structural_iterator.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for haswell */ /* including generic/stage2/tape_builder.h for haswell: #include */ /* begin file generic/stage2/tape_builder.h for haswell */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -21489,12 +28891,8 @@ namespace haswell { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -21547,88 +28945,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -21637,27 +29077,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -21665,7 +29126,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -21679,76 +29141,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -21761,13 +29289,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -22123,16 +29653,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace icelake @@ -22888,6 +30408,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -22899,6 +30422,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -22935,6 +30480,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace icelake @@ -23025,7 +30635,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -23204,6 +30814,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -23243,6 +30854,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -23499,6 +31122,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -23536,6 +31372,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -23554,6 +31400,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -23570,26 +31438,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -23678,7 +31593,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -23761,15 +31676,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -23800,7 +31717,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -23849,7 +31780,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -23948,7 +31879,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24046,7 +31977,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24101,7 +32032,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24187,7 +32118,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24227,9 +32158,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -24239,12 +32408,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -24276,7 +32537,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -24293,163 +32554,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + return INCORRECT_TYPE; } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -24460,9 +32597,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -24501,9 +32637,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -24866,16 +33002,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace icelake @@ -26428,6 +34554,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace icelake @@ -26861,7 +35277,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -26889,6 +35304,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -26971,7 +35455,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -27268,7 +35751,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -27285,6 +35768,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -27293,6 +35777,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -27341,7 +35826,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -27356,7 +35841,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -27368,8 +35853,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -27386,29 +35871,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -27436,16 +35921,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -27473,11 +35958,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -27517,7 +36020,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -27539,7 +36055,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -27725,9 +36254,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -27750,6 +36283,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -27804,73 +36408,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for icelake */ -/* including generic/stage2/structural_iterator.h for icelake: #include */ -/* begin file generic/stage2/structural_iterator.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for icelake */ /* including generic/stage2/tape_builder.h for icelake: #include */ /* begin file generic/stage2/tape_builder.h for icelake */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -27893,12 +36430,8 @@ namespace icelake { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -27951,88 +36484,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -28041,27 +36616,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -28069,7 +36665,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -28083,76 +36680,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -28165,13 +36828,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -28542,16 +37207,6 @@ simdjson_inline int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace ppc64 @@ -29450,6 +38105,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -29461,6 +38119,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -29497,6 +38177,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace ppc64 @@ -29587,7 +38332,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -29766,6 +38511,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -29805,6 +38551,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -30061,6 +38819,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -30098,6 +39069,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -30116,6 +39097,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -30132,26 +39135,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -30240,7 +39290,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -30323,15 +39373,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -30362,7 +39414,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -30411,7 +39477,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -30510,7 +39576,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30608,7 +39674,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30663,7 +39729,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30749,7 +39815,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30789,11 +39855,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -30804,9 +39879,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -30855,6 +39929,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -31007,11 +40172,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -31022,9 +40201,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -31073,6 +40251,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -31398,16 +40669,6 @@ simdjson_inline int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace ppc64 @@ -33103,6 +42364,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace ppc64 @@ -33536,7 +43087,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -33564,6 +43114,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -33646,7 +43265,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -33943,7 +43561,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -33960,6 +43578,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -33968,6 +43587,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -34016,7 +43636,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -34031,7 +43651,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -34043,8 +43663,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -34061,29 +43681,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -34111,16 +43731,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -34148,11 +43768,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -34192,7 +43830,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -34214,7 +43865,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -34400,9 +44064,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -34425,6 +44093,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -34479,73 +44218,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for ppc64 */ -/* including generic/stage2/structural_iterator.h for ppc64: #include */ -/* begin file generic/stage2/structural_iterator.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for ppc64 */ /* including generic/stage2/tape_builder.h for ppc64: #include */ /* begin file generic/stage2/tape_builder.h for ppc64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -34568,12 +44240,8 @@ namespace ppc64 { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -34626,88 +44294,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -34716,27 +44426,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -34744,7 +44475,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -34758,76 +44490,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -34840,13 +44638,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -35161,16 +44961,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -35749,16 +45539,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -36372,6 +46152,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -36383,6 +46166,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -36419,6 +46224,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace westmere @@ -36509,7 +46379,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -36688,6 +46558,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -36727,6 +46598,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -36983,6 +46866,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -37020,6 +47116,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -37038,6 +47144,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -37054,26 +47182,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -37162,7 +47337,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -37245,15 +47420,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -37284,7 +47461,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -37333,7 +47524,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -37432,7 +47623,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37530,7 +47721,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37585,7 +47776,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37671,7 +47862,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37711,9 +47902,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -37723,12 +48152,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37760,7 +48281,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -37777,163 +48298,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -37944,9 +48341,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37985,9 +48381,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -38332,16 +48728,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -38920,16 +49306,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -40340,6 +50716,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace westmere @@ -40773,7 +51439,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -40801,6 +51466,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -40883,7 +51617,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -41180,7 +51913,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -41197,6 +51930,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -41205,6 +51939,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -41253,7 +51988,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -41268,7 +52003,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -41280,8 +52015,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -41298,29 +52033,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -41348,16 +52083,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -41385,11 +52120,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -41429,7 +52182,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -41451,7 +52217,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -41637,9 +52416,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -41662,6 +52445,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -41716,73 +52570,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for westmere */ -/* including generic/stage2/structural_iterator.h for westmere: #include */ -/* begin file generic/stage2/structural_iterator.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for westmere */ /* including generic/stage2/tape_builder.h for westmere: #include */ /* begin file generic/stage2/tape_builder.h for westmere */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -41805,12 +52592,8 @@ namespace westmere { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -41863,88 +52646,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -41953,27 +52778,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -41981,7 +52827,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -41995,76 +52842,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -42077,13 +52990,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -42390,10 +53305,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lasx @@ -43140,6 +54051,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -43151,6 +54065,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -43187,6 +54123,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lasx @@ -43277,7 +54278,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -43456,6 +54457,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -43495,6 +54497,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -43751,6 +54765,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -43788,6 +55015,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -43806,6 +55043,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -43822,26 +55081,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -43930,7 +55236,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -44013,15 +55319,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -44052,7 +55360,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -44101,7 +55423,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -44200,7 +55522,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44298,7 +55620,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44353,7 +55675,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44439,7 +55761,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44479,11 +55801,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -44494,9 +55825,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -44545,6 +55875,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -44697,11 +56118,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -44712,9 +56147,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -44763,6 +56197,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -45061,10 +56588,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lasx @@ -46608,6 +58131,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace lasx @@ -47041,7 +58854,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -47069,6 +58881,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -47151,7 +59032,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -47448,7 +59328,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -47465,6 +59345,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -47473,6 +59354,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -47521,7 +59403,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -47536,7 +59418,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -47548,8 +59430,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -47566,29 +59448,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -47616,16 +59498,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -47653,11 +59535,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -47697,7 +59597,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -47719,7 +59632,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -47905,9 +59831,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -47930,6 +59860,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -47984,73 +59985,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for lasx */ -/* including generic/stage2/structural_iterator.h for lasx: #include */ -/* begin file generic/stage2/structural_iterator.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for lasx */ /* including generic/stage2/tape_builder.h for lasx: #include */ /* begin file generic/stage2/tape_builder.h for lasx */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -48073,12 +60007,8 @@ namespace lasx { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -48131,88 +60061,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -48221,27 +60193,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -48249,7 +60242,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -48263,76 +60257,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -48345,13 +60405,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -48613,10 +60675,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lsx @@ -49345,6 +61403,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -49356,6 +61417,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -49392,6 +61475,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lsx @@ -49482,7 +61630,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -49661,6 +61809,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -49700,6 +61849,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -49956,6 +62117,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -49993,6 +62367,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -50011,6 +62395,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -50027,26 +62433,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -50135,7 +62588,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -50218,15 +62671,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -50257,7 +62712,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -50306,7 +62775,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -50405,7 +62874,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50503,7 +62972,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50558,7 +63027,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50644,7 +63113,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50684,9 +63153,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -50696,12 +63403,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -50733,7 +63532,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -50750,163 +63549,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - exponent += exp_neg ? 0-exp : exp; + return INCORRECT_TYPE; } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -50917,9 +63592,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -50958,9 +63632,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -51251,10 +63925,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lsx @@ -52780,6 +65450,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace lsx @@ -53213,7 +66173,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -53241,6 +66200,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -53323,7 +66351,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -53620,7 +66647,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -53637,6 +66664,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -53645,6 +66673,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -53693,7 +66722,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -53708,7 +66737,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -53720,8 +66749,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -53738,29 +66767,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -53788,16 +66817,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -53825,11 +66854,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -53869,7 +66916,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -53891,7 +66951,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -54077,9 +67150,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -54102,6 +67179,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -54156,73 +67304,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for lsx */ -/* including generic/stage2/structural_iterator.h for lsx: #include */ -/* begin file generic/stage2/structural_iterator.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for lsx */ /* including generic/stage2/tape_builder.h for lsx: #include */ /* begin file generic/stage2/tape_builder.h for lsx */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -54245,12 +67326,8 @@ namespace lsx { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -54303,88 +67380,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -54393,27 +67512,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -54421,7 +67561,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -54435,76 +67576,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -54517,13 +67724,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -54793,11 +68002,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { return __builtin_popcountll(input_num); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace rvv_vls @@ -55538,6 +68742,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -55549,6 +68756,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -55585,6 +68814,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace rvv_vls @@ -55675,7 +68969,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -55854,6 +69148,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -55893,6 +69188,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -56149,6 +69456,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -56186,6 +69706,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -56204,6 +69734,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -56220,26 +69772,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -56328,7 +69927,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -56411,15 +70010,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -56450,7 +70051,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -56499,7 +70114,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -56598,7 +70213,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56696,7 +70311,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56751,7 +70366,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56837,7 +70452,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56877,11 +70492,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -56892,9 +70516,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -56943,6 +70566,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -57095,11 +70809,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -57110,9 +70838,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -57161,6 +70888,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -57339,7 +71159,7 @@ class implementation final : public simdjson::implementation { simdjson_inline implementation() : simdjson::implementation( "rvv_vls", "RISC-V V extension", - 0 + internal::instruction_set::RVV_VLS ) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, @@ -57456,11 +71276,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { return __builtin_popcountll(input_num); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace rvv_vls @@ -59371,6 +73186,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace rvv_vls @@ -59804,7 +73909,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -59832,6 +73936,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -59914,7 +74087,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -60211,7 +74383,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -60228,6 +74400,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -60236,6 +74409,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -60284,7 +74458,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -60299,7 +74473,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -60311,8 +74485,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -60329,29 +74503,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -60379,16 +74553,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -60416,11 +74590,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -60460,7 +74652,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -60482,7 +74687,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -60668,9 +74886,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -60693,6 +74915,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -60747,73 +75040,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for rvv_vls */ -/* including generic/stage2/structural_iterator.h for rvv_vls: #include */ -/* begin file generic/stage2/structural_iterator.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for rvv_vls */ /* including generic/stage2/tape_builder.h for rvv_vls: #include */ /* begin file generic/stage2/tape_builder.h for rvv_vls */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -60836,12 +75062,8 @@ namespace rvv_vls { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -60894,88 +75116,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -60984,27 +75248,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -61012,7 +75297,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -61026,76 +75312,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -61108,13 +75460,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -61708,6 +76062,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -61719,6 +76076,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -61755,6 +76134,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace fallback @@ -61845,7 +76289,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -62024,6 +76468,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -62063,6 +76508,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -62319,6 +76776,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -62356,6 +77026,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -62374,6 +77054,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -62390,26 +77092,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -62498,7 +77247,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -62581,15 +77330,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -62620,7 +77371,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -62669,7 +77434,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -62768,7 +77533,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -62866,7 +77631,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -62921,7 +77686,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -63007,7 +77772,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -63047,9 +77812,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -63059,12 +78062,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -63096,7 +78191,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -63113,163 +78208,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -63280,9 +78251,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -63321,9 +78291,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -63871,6 +78841,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace fallback @@ -64050,9 +79310,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -64075,6 +79339,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -64278,7 +79613,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -64295,6 +79630,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -64303,6 +79639,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -64351,7 +79688,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -64366,7 +79703,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -64378,8 +79715,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -64396,29 +79733,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -64446,16 +79783,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -64483,11 +79820,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -64527,7 +79882,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -64549,7 +79917,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -64720,12 +80101,8 @@ namespace fallback { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -64778,88 +80155,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -64868,27 +80287,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -64896,7 +80336,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -64910,76 +80351,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -64992,13 +80499,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -65229,14 +80738,14 @@ simdjson_warn_unused simdjson_inline error_code scan() { // Primitive or invalid character (invalid characters will be checked in stage 2) } else { // Anything else, add the structural and go until we find the next one. - // We also stop on '"' so that an unclosed string still reaches - // validate_string(); a quote swallowed by the run would hide it. A - // quote cannot occur inside a valid primitive. We deliberately do not - // stop on every ESC_ASCII character: that also covers a backslash and the - // control characters, and ending the run there makes the fallback - // disagree with the SIMD kernels. + // We also stop on RS (0x1E) so that RFC 7464 json_sequence inputs + // like `\x1e"a"\x1e"b"` produce a separate structural for each RS + // rather than being absorbed into a single primitive run, and on '"' + // so that an unclosed string still reaches validate_string(). Neither + // can occur inside a valid primitive. add_structural(); - while (idx+1= 202402L) // update when the standard is finalized +// While C++26 is a working draft, compilers report 202400L in C++26 mode +// (both GCC 16 and Clang 21 do). Update when the standard is finalized. +#if !defined(SIMDJSON_CPLUSPLUS26) && (SIMDJSON_CPLUSPLUS >= 202400L) #define SIMDJSON_CPLUSPLUS26 1 #endif @@ -118,14 +120,48 @@ #endif #endif -// The current specification is unclear on how we detect -// static reflection, both __cpp_lib_reflection and -// __cpp_impl_reflection are proposed in the draft specification. -// For now, we disable static reflect by default. It must be -// specified at compiler time. +// Static reflection. +// +// The reflection-based APIs (simdjson::to, document::get, the builder, +// compile-time JSON, annotations) need considerably more than the reflection +// operator. We turn them on only when the compiler advertises all of: +// +// P2996 reflection (^^, splicers, ) __cpp_impl_reflection, +// __cpp_lib_reflection +// P1306 expansion statements (template for) __cpp_expansion_statements +// P3491 std::define_static_string / _array __cpp_lib_define_static +// +// Two further features we rely on have, as of this writing, no feature-test +// macro of their own, so they cannot be checked directly: +// +// P3394 annotations ([[=x]], std::meta::annotations_of) -- used for +// the annotations of simdjson/annotations.h (rename, skip, ...). +// P3289 consteval blocks (consteval { ... }) -- used by compile_time_json. +// +// Every implementation that defines the four macros above also implements +// those two, so requiring the four is sufficient in practice. If that ever +// stops being true, define SIMDJSON_STATIC_REFLECTION=0 to opt out. +// +// SIMDJSON_STATIC_REFLECTION may always be defined by the user (or by the +// build system) to 0 or 1 to override the detection. +// +// Note that C++26 mode alone is not enough: GCC 16 requires -freflection, +// and only then does it define __cpp_impl_reflection. #ifndef SIMDJSON_STATIC_REFLECTION -#define SIMDJSON_STATIC_REFLECTION 0 // disabled by default. +#if defined(SIMDJSON_CPLUSPLUS26) && \ + defined(__cpp_impl_reflection) && __cpp_impl_reflection >= 202506L && \ + defined(__cpp_lib_reflection) && __cpp_lib_reflection >= 202506L && \ + defined(__cpp_expansion_statements) && \ + __cpp_expansion_statements >= 202506L && \ + defined(__cpp_lib_define_static) && __cpp_lib_define_static >= 202506L +// __cpp_lib_reflection is the feature-test macro for , so there is no +// need for a separate __has_include check (which would have to be guarded for +// compilers that lack __has_include). +#define SIMDJSON_STATIC_REFLECTION 1 +#else +#define SIMDJSON_STATIC_REFLECTION 0 #endif +#endif // SIMDJSON_STATIC_REFLECTION #if defined(__apple_build_version__) #if __apple_build_version__ < 14000000 @@ -158,6 +194,47 @@ #define SIMDJSON_SUPPORTS_DESERIALIZATION 0 #endif +// The C++20 char8_t type (and std::u8string/std::u8string_view) is available. +// Because all strings that simdjson produces are valid UTF-8, we can offer +// char8_t variants of our string accessors when this macro is set. +#if !defined(SIMDJSON_SUPPORTS_CHAR8_T) +#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L +#define SIMDJSON_SUPPORTS_CHAR8_T 1 +#else +#define SIMDJSON_SUPPORTS_CHAR8_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_CHAR8_T) + +// The C++23 fixed-width floating-point types std::float32_t and std::float64_t +// () are available. They are optional even in C++23: a compiler that +// provides them predefines __STDCPP_FLOAT32_T__ and __STDCPP_FLOAT64_T__. +// When these macros are set, we offer get_float32() and get_float64(). +#if !defined(SIMDJSON_SUPPORTS_FLOAT32_T) +#if defined(__STDCPP_FLOAT32_T__) && defined(__has_include) +#if __has_include() +#define SIMDJSON_SUPPORTS_FLOAT32_T 1 +#endif +#endif +#ifndef SIMDJSON_SUPPORTS_FLOAT32_T +#define SIMDJSON_SUPPORTS_FLOAT32_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_FLOAT32_T) + +#if !defined(SIMDJSON_SUPPORTS_FLOAT64_T) +#if defined(__STDCPP_FLOAT64_T__) && defined(__has_include) +#if __has_include() +#define SIMDJSON_SUPPORTS_FLOAT64_T 1 +#endif +#endif +#ifndef SIMDJSON_SUPPORTS_FLOAT64_T +#define SIMDJSON_SUPPORTS_FLOAT64_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_FLOAT64_T) + +#if SIMDJSON_SUPPORTS_FLOAT32_T || SIMDJSON_SUPPORTS_FLOAT64_T +#include +#endif + #if !defined(SIMDJSON_CONSTEVAL) #if defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L @@ -166,6 +243,18 @@ #define SIMDJSON_CONSTEVAL 0 #endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #endif // !defined(SIMDJSON_CONSTEVAL) + +// SIMDJSON_CONSTEXPR_STRING is 'constexpr' when the standard library supports +// constexpr std::string (e.g., libstdc++ 12 or better), and empty otherwise. It +// lets functions that build a std::string be constant expressions when possible +// while still compiling against older standard libraries. +#if !defined(SIMDJSON_CONSTEXPR_STRING) +#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L +#define SIMDJSON_CONSTEXPR_STRING constexpr +#else +#define SIMDJSON_CONSTEXPR_STRING +#endif // defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L +#endif // !defined(SIMDJSON_CONSTEXPR_STRING) #endif // SIMDJSON_COMPILER_CHECK_H /* end file simdjson/compiler_check.h */ /* including simdjson/portability.h: #include "simdjson/portability.h" */ @@ -457,16 +546,86 @@ using std::size_t; #endif #endif +#ifndef SIMDJSON_HAS_UNISTD_H +#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) +#define SIMDJSON_HAS_UNISTD_H 1 +#else +#define SIMDJSON_HAS_UNISTD_H 0 +#endif +#endif + +// padded_memory_map availability. +// +// On POSIX platforms the class is always available: the implementation uses +// `mmap` (and a trailing anonymous page for padding) from . +// +// On Windows the class is disabled by default and must be explicitly +// opted into by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. Enabling +// it requires: +// 1. `` has been included *before* `` (so that +// this header can see the Win32 types and the `_WINDOWS_` include +// guard), +// 2. the compilation targets Windows 10, version 1803 or later +// (i.e. `NTDDI_VERSION >= NTDDI_WIN10_RS4`, `0x0A000005`). This is +// required because the implementation relies on the modern memory +// APIs introduced with that version (`CreateFileMapping2` / +// `MapViewOfFile3`), +// 3. the link step pulls in an import library that exports those APIs, +// typically `onecore.lib` (or `mincore.lib`). +// +// The `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS` CMake option arranges (1)-(3) +// automatically when building simdjson with its own CMake. Consumers using +// simdjson as a pre-built library are responsible for setting the macro, +// the Windows version macros, and the link library themselves. +// +// If the opt-in conditions are not met on Windows, `padded_memory_map` +// simply does not exist -- any attempt to use it fails at compile time +// with an "unknown identifier" diagnostic rather than silently degrading. +// +// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is +// available in the current translation unit. Users may test this macro to +// conditionally compile code that depends on padded_memory_map. +#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP + #if defined(__unix__) || defined(__APPLE__) || defined(__linux__) + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #elif defined(_WINDOWS_) && defined(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) && SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #else + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 0 + #endif +#endif #endif // SIMDJSON_PORTABILITY_H /* end file simdjson/portability.h */ +#include namespace simdjson { namespace internal { +/** + * @private + * Scratch capacity that every caller of to_chars must provide. + * + * The emitted decimal is at most ~24 characters, but dragonbox() and + * format_buffer() intentionally write past the logical end with fixed-size + * 16/17-byte memcpy/memset operations so the compiler can inline them (no + * libc mem* dispatch with size-class branches). The extra bytes are required + * for safety of those over-writes; do not shrink this below 40. + * See src/to_chars.cpp and #2805. + */ +// Use an unscoped enum (not static constexpr / inline constexpr): +// - C++11 targets (readme_examples11, quickstart11, ...) still include this header +// - a static constexpr in the amalgamated simdjson.cpp TU is unused there +// (only callers in headers use it) and trips -Wunused-const-variable -Werror +enum : size_t { to_chars_buffer_size = 40 }; /** * @private * Our own implementation of the C++17 to_chars function. * Defined in src/to_chars + * + * @note The buffer starting at first must have at least to_chars_buffer_size + * bytes of writable storage (see to_chars_buffer_size). + * @note The input number must be finite (NaN/Inf are not supported). + * @note The result is NOT null-terminated. */ char *to_chars(char *first, const char *last, double value); /** @@ -476,6 +635,12 @@ char *to_chars(char *first, const char *last, double value); */ double from_chars(const char *first) noexcept; double from_chars(const char *first, const char* end) noexcept; +/** + * @private + * Same as from_chars, but produces a correctly rounded binary32 (float) value. + * Defined in src/from_chars + */ +float from_chars_float(const char *first) noexcept; } #ifndef SIMDJSON_EXCEPTIONS @@ -486,6 +651,10 @@ double from_chars(const char *first, const char* end) noexcept; #endif #endif +#ifndef SIMDJSON_ENABLE_NAN_INF +#define SIMDJSON_ENABLE_NAN_INF 0 +#endif + } // namespace simdjson #if defined(__GNUC__) @@ -501,16 +670,14 @@ double from_chars(const char *first, const char* end) noexcept; // Align to N-byte boundary #define SIMDJSON_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) -#define SIMDJSON_ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) - -#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #if SIMDJSON_REGULAR_VISUAL_STUDIO // We could use [[deprecated]] but it requires C++14 #define simdjson_deprecated __declspec(deprecated) #define simdjson_really_inline __forceinline - #define simdjson_never_inline __declspec(noinline) + #define simdjson_never_inline inline __declspec(noinline) + #define simdjson_really_flatten [[msvc::flatten]] #define simdjson_unused #define simdjson_warn_unused @@ -551,6 +718,7 @@ double from_chars(const char *first, const char* end) noexcept; #define simdjson_really_inline inline __attribute__((always_inline)) #define simdjson_never_inline inline __attribute__((noinline)) + #define simdjson_really_flatten [[gnu::flatten]] #define simdjson_unused __attribute__((unused)) #define simdjson_warn_unused __attribute__((warn_unused_result)) @@ -627,6 +795,15 @@ double from_chars(const char *first, const char* end) noexcept; #define simdjson_inline simdjson_really_inline #endif +#if defined(simdjson_flatten) + // Prefer the user's definition of simdjson_flatten; don't define it ourselves. +#elif (defined(__GNUC__) && !defined(__OPTIMIZE__)) || (defined(_DEBUG) && _MSC_VER ) + // Flattening can lead to significant code bloat and high compile times. Don't use it for unoptimized builds. + #define simdjson_flatten +#else + #define simdjson_flatten simdjson_really_flatten +#endif + #if SIMDJSON_VISUAL_STUDIO /** * Windows users need to do some extra work when building @@ -2538,22 +2715,22 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "4.6.11" +#define SIMDJSON_VERSION "5.0.3" namespace simdjson { enum { /** * The major version (MAJOR.minor.revision) of simdjson being used. */ - SIMDJSON_VERSION_MAJOR = 4, + SIMDJSON_VERSION_MAJOR = 5, /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 6, + SIMDJSON_VERSION_MINOR = 0, /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 11 + SIMDJSON_VERSION_REVISION = 3 }; } // namespace simdjson @@ -2625,6 +2802,7 @@ enum error_code { OUT_OF_BOUNDS, ///< Attempted to access location outside of document. TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory. + UNKNOWN_FIELD, ///< JSON field does not map to any member of the target (see simdjson::deny_unknown_fields) NUM_ERROR_CODES ///< Placeholder for end of error code list. }; @@ -2987,6 +3165,7 @@ inline const std::string error_message(int error) noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS #include +#include #include namespace simdjson { @@ -3028,6 +3207,19 @@ concept constructible_from_string_view = std::is_constructible_v && std::is_default_constructible_v; +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * A C++20 char8_t string type such as std::u8string. Such types cannot be built + * from a std::string_view (the character types differ), so they need their own + * deserialization path, going through the u8 string accessors. + */ +template +concept constructible_from_u8string_view = std::is_constructible_v + && !std::is_same_v + && !std::is_constructible_v + && std::is_default_constructible_v; +#endif // SIMDJSON_SUPPORTS_CHAR8_T + template concept string_view_keyed_map = string_view_like && requires(std::remove_cvref_t& m, typename M::key_type sv, typename M::mapped_type v) { @@ -3122,9 +3314,15 @@ concept string_like = // Concept that checks if a type is a container but not a string (because // strings handling must be handled differently) // Now uses iterator-based approach for broader container support +// +// Optional types are excluded on purpose. Since C++26 (P3168), std::optional +// is itself a range, so without the exclusion an std::optional would match +// both this concept and optional_type, making the container and the optional +// overloads of atom()/append() ambiguous. See issue 2827. template concept container_but_not_string = - std::ranges::input_range && !string_like && !concepts::string_view_keyed_map; + std::ranges::input_range && !string_like && !concepts::string_view_keyed_map + && !concepts::optional_type; @@ -3231,6 +3429,11 @@ struct fixed_string { data[i] = str[i]; } } + constexpr fixed_string(const unsigned char (&str)[N]) { + for (std::size_t i = 0; i < N; ++i) { + data[i] = static_cast(str[i]); + } + } char data[N]; constexpr std::string_view view() const { return {data, N - 1}; } constexpr size_t size() const { return N ; } @@ -3262,6 +3465,11 @@ struct string_constant { #endif // SIMDJSON_CONSTEVALUTIL_H /* end file simdjson/constevalutil.h */ +#if SIMDJSON_SUPPORTS_CHAR8_T +#include +#include +#endif + /** * @brief The top level simdjson namespace, containing everything the library provides. */ @@ -3271,6 +3479,10 @@ SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS /** The maximum document size supported by simdjson. */ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; +/** The maximum depth of nested objects and arrays supported by simdjson. + A depth of SIMDJSON_MAXSIZE_BYTES/2 is not reasonable and would be + adversarial, but it serves as an upper bound for validation purposes. */ +constexpr size_t SIMDJSON_MAX_DEPTH = SIMDJSON_MAXSIZE_BYTES/2; /** * The amount of padding needed in a buffer to parse JSON. @@ -3296,6 +3508,30 @@ struct padded_string; class padded_string_view; enum class stage1_mode; +/** + * Stream format for parse_many/iterate_many. + */ +enum class stream_format { + whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL) + json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited) + comma_delimited, ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`) + comma_delimited_array,///< A single JSON array whose elements are iterated as + ///< comma-separated documents (e.g., `[{...},{...},{...}]`). + ///< The parser strips the outer `[` / `]` plus any + ///< surrounding JSON whitespace (space, tab, LF, CR) + ///< and then behaves like `comma_delimited` over the + ///< remaining bytes. + newline_delimited ///< NDJSON/JSON Lines where each document occupies exactly + ///< one line: documents are separated by line feeds and no + ///< document contains a raw line feed. Same inputs as + ///< `whitespace_delimited`, but the stronger guarantee lets + ///< the parser find the end of a document without walking + ///< it. On ondemand `iterate_many`, an unread remainder may + ///< be skipped by jumping to the next line feed without + ///< structure-validating that remainder. Use + ///< `whitespace_delimited` if unsure. +}; + namespace internal { template @@ -3306,6 +3542,52 @@ class tape_ref; struct value128; enum class tape_type; +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * Reinterpret a UTF-8 string as a C++20 std::u8string_view. No byte is copied + * or modified: char8_t and char have the same size, representation and + * alignment. Every string that simdjson produces is valid UTF-8, so this is a + * lossless view over the very same memory. + * @private + */ +simdjson_inline std::u8string_view as_u8string_view(std::string_view v) noexcept { + return std::u8string_view(reinterpret_cast(v.data()), v.size()); +} +#endif // SIMDJSON_SUPPORTS_CHAR8_T + +/** + * Assign a UTF-8 string to a string-like receiver. The general case simply + * assigns the std::string_view: it covers std::string and any user type that + * can be assigned from a std::string_view. + * @private + */ +template +simdjson_inline void assign_utf8(string_type &receiver, std::string_view content) noexcept { + receiver = content; +} + +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * Assign a UTF-8 string to a char8_t-based string (e.g., std::u8string). This + * overload is more specialized than the general one, so overload resolution + * prefers it whenever the receiver holds char8_t. + * @private + */ +template +simdjson_inline void assign_utf8(std::basic_string &receiver, std::string_view content) noexcept { + receiver.assign(reinterpret_cast(content.data()), content.size()); +} + +/** + * Assign a UTF-8 string to a char8_t-based string view (e.g., std::u8string_view). + * @private + */ +template +simdjson_inline void assign_utf8(std::basic_string_view &receiver, std::string_view content) noexcept { + receiver = std::basic_string_view(reinterpret_cast(content.data()), content.size()); +} +#endif // SIMDJSON_SUPPORTS_CHAR8_T + } // namespace internal } // namespace simdjson @@ -3599,7 +3881,12 @@ class document; * 3) The stream_final mode allows us to truncate final * unterminated strings. It is useful in conjunction with streaming_partial. */ -enum class stage1_mode { regular, streaming_partial, streaming_final}; +enum class stage1_mode { + regular, + streaming_partial, streaming_final, + json_sequence_partial, json_sequence_final, + comma_delimited_partial, comma_delimited_final +}; /** * Returns true if mode == streaming_partial or mode == streaming_final @@ -3611,7 +3898,6 @@ inline bool is_streaming(stage1_mode mode) { // return (mode == stage1_mode::streaming_partial || mode == stage1_mode::streaming_final); } - namespace internal { @@ -3796,6 +4082,16 @@ class dom_parser_implementation { /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ bool _number_as_string{false}; + /** + * Whether the input buffer passed to parse() is *not* padded to len + + * SIMDJSON_PADDING bytes. When true, stage 2 string parsing avoids reading + * past buf+len (it finishes the final, near-the-end bytes from a small padded + * scratch buffer). This is set only by the no-padding DOM parse entry points + * (dom::parser::parse_unpadded); the default padded fast path leaves it false + * and is unaffected. + */ + bool _unpadded{false}; + protected: // Declaring these so that subclasses can use them to implement their constructors. @@ -4350,11 +4646,26 @@ inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { ret inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } #endif - -#ifndef _WIN32 +#if SIMDJSON_HAS_PADDED_MEMORY_MAP /** * A class representing a memory-mapped file with padding. - * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. + * + * On POSIX systems (Linux, macOS, BSD, ...), this uses `mmap` to map the file + * contents directly into memory, which is efficient for large files (no copy). + * + * On Windows, this class is disabled by default and must be opted into at + * build time by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. When + * enabled, `` must also be included before `` and + * the compilation must target Windows 10, version 1803 or later. The + * Windows implementation uses the modern memory APIs (`VirtualAlloc2`, + * `CreateFileMapping2`, `MapViewOfFile3`) with the placeholder virtual + * memory mechanism to always achieve true zero-copy mapping with + * contiguous zero-filled padding. + * + * Either way, the resulting `padded_string_view` carries at least + * `SIMDJSON_PADDING` bytes of accessible zero-filled padding after the file + * content, so it can be consumed directly by the simdjson parsers (including + * `parse_many` / `iterate_many`). */ class padded_memory_map { public: @@ -4362,9 +4673,11 @@ class padded_memory_map { * Create a new padded memory map for the given file. * After creating the memory map, you can call view() to get a padded_string_view of the file content. * The memory map will be automatically released when the padded_memory_map instance is destroyed. - * Note that the file content is not copied, so this is efficient for large files. However, - * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, - * permission denied, etc.), the memory map will be invalid and view() will return an empty view. + * On POSIX systems, the file content is not copied, so this is efficient for large files. + * On Windows, the file is mapped into memory via `MapViewOfFile3` (zero-copy). + * In all cases, the file must remain unchanged while the memory map is in use. + * In case of error (e.g., file not found, permission denied, etc.), the memory map will be + * invalid and view() will return an empty view. * You can check if the memory map is valid by calling is_valid() before using view(). * * @param filename the path to the file to memory-map. @@ -4401,8 +4714,14 @@ class padded_memory_map { padded_memory_map &operator=(const padded_memory_map &) = delete; const char *data{nullptr}; size_t size{0}; +#ifdef _WIN32 + // When the file ends near an allocation-granularity boundary, we use the + // placeholder API to append zero-filled padding pages. This pointer tracks + // that region so the destructor can release it with VirtualFree. + void *padding_view_{nullptr}; +#endif }; -#endif // _WIN32 +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP @@ -4476,6 +4795,9 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s #include #include #include +#if SIMDJSON_CPLUSPLUS17 +#include +#endif namespace simdjson { @@ -4540,6 +4862,68 @@ class padded_string_view : public std::string_view { }; // padded_string_view +/** + * Get the system's memory page size. By default, we return + * 4096 bytes, which is the most common page size. On systems + * where the page size is not a multiple of 4096 bytes, and not + * a unix-like system, nor Windows, this function may return an + * incorrect value. + * + * @return The page size in bytes. + */ +inline uint32_t get_page_size() noexcept; + +#if SIMDJSON_CPLUSPLUS17 +/** + * A padded_input is a wrapper around either a padded_string_view or a padded_string. + * It will automatically pad a string_view if it does not have sufficient padding + * up to the end of the memory page. Note that a requirement for this method to + * make sense is to be on a system with a page size of at least 4096 (which is + * universal except on some embedded systems). + */ +struct padded_input { + /** + * Construct a padded_input from a string_view. If the string_view does not have sufficient padding, + * the data will be copied into a padded_string and the padded_string_view will point to the + * padded_string's data. Otherwise, the padded_string_view will point to the original string_view's data. + */ + inline explicit padded_input(std::string_view sv); + /** + * Construct a padded_input from a C-style string (length specified). If the string does not have sufficient padding, + * the data will be copied into a padded_string and the padded_string_view will point to the + * padded_string's data. Otherwise, the padded_string_view will point to the original string's data. + */ + inline explicit padded_input(const char *data, size_t length); + /** + * Construct a padded_input from a std::string. If the string does not have sufficient padding + * (considering its capacity), the data will be copied into a padded_string and the padded_string_view + * will point to the padded_string's data. Otherwise, the padded_string_view will point to the + * original string's data. + */ + inline explicit padded_input(const std::string &s); + + /** + * Check if the padded_input is a view. + * + * @return true if the padded_input is a view, false otherwise. + */ + inline bool is_view() const noexcept; + + /** + * Convert the padded_input to a padded_string_view. + * + * @return The padded_string_view. + */ + inline operator simdjson::padded_string_view() const noexcept; + +private: + std::variant storage; + // whether we cross a page boundary and need to allocate a new padded string. + static inline bool needs_allocation(const char* buf, size_t len, size_t padding = SIMDJSON_PADDING) noexcept; +}; + +#endif // SIMDJSON_CPLUSPLUS17 + #if SIMDJSON_EXCEPTIONS /** * Send padded_string instance to an output stream. @@ -4572,6 +4956,31 @@ inline padded_string_view pad(std::string& s) noexcept; * @return The padded string. */ inline padded_string_view pad_with_reserve(std::string& s) noexcept; + +/** + * Return the index-th document-aligned slice of a delimited stream. + * + * The input is divided into blocks of block_size bytes and each boundary is + * moved forward to just past the next delimiter, so a document is never split. + * The delimiter must not occur inside a document: a line feed for NDJSON, a + * record separator (0x1E) for RFC 7464. + * + * Slices are contiguous and non-overlapping, and each may be parsed + * independently, so callers can process them on as many threads as they like. + * + * Iterate while index * block_size < data.size(). A slice is empty when its + * block falls entirely inside one document, which happens only if that document + * is longer than block_size; skip it and continue. With block_size larger than + * the longest document, no slice is ever empty. + * + * @param data The padded input. + * @param delimiter The byte that separates documents. + * @param block_size The nominal slice size, before snapping. + * @param index Which slice to return, counting from zero. + */ +inline padded_string_view slice_at(padded_string_view data, char delimiter, + size_t block_size, size_t index) noexcept; + } // namespace simdjson #endif // SIMDJSON_PADDED_STRING_VIEW_H @@ -4588,6 +4997,15 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept; #include /* memcmp */ +// for page size computation. +#if SIMDJSON_HAS_UNISTD_H + #include + #if defined(__APPLE__) + #include + #endif +#endif + + namespace simdjson { inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept @@ -4677,7 +5095,102 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept { return padded_string_view(s.data(), s.size(), s.capacity()); } +inline uint32_t get_page_size() noexcept { +#if defined(_WINDOWS_) // if and only if someone loaded Windows.h, we can get the page size from there. +// Otherwise, we assume 4096. + static const uint32_t cached = []() -> uint32_t { + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwPageSize); + }(); + return cached; +#elif SIMDJSON_HAS_UNISTD_H + static const uint32_t cached = []() -> uint32_t { + long page_size = sysconf(_SC_PAGESIZE); + if (page_size > 0) { + return static_cast(page_size); + } + return 4096; // fallback + }(); + return cached; +#else + return 4096; // fallback +#endif +} +#if SIMDJSON_CPLUSPLUS17 + +inline padded_input::padded_input(std::string_view sv) + : storage(simdjson::padded_string_view{}) { + if (needs_allocation(sv.data(), sv.size())) { + storage = simdjson::padded_string(sv); + } else { + storage = simdjson::padded_string_view( + sv.data(), sv.size(), sv.size() + simdjson::SIMDJSON_PADDING); + } +} + +inline padded_input::padded_input(const char *data, size_t length) + : storage(simdjson::padded_string_view{}) { + if (needs_allocation(data, length)) { + storage = simdjson::padded_string(data, length); + } else { + storage = simdjson::padded_string_view( + data, length, length + simdjson::SIMDJSON_PADDING); + } +} +inline padded_input::padded_input(const std::string &s) + : storage(simdjson::padded_string_view{}) { + const size_t len = s.size(); + const size_t cap = s.capacity(); + // Here we have the string content from data() to data() + size(), + // but the memory is accessible from data() to data() + capacity(). + const size_t needed_padding = (cap - len) < simdjson::SIMDJSON_PADDING + ? simdjson::SIMDJSON_PADDING - (cap - len) : 0; + if (needed_padding > 0 && needs_allocation(s.data(), cap, needed_padding)) { + storage = simdjson::padded_string(s); + } else { + storage = simdjson::padded_string_view( + s.data(), len, len + simdjson::SIMDJSON_PADDING); + } +} + +inline bool padded_input::is_view() const noexcept { + return std::holds_alternative(storage); +} + +inline padded_input::operator simdjson::padded_string_view() const noexcept { + return std::visit([](const auto& p) -> simdjson::padded_string_view { + return p; + }, storage); +} + +inline bool padded_input::needs_allocation(const char* buf, size_t len, size_t padding) noexcept { + if(len == 0) { return false; } + const auto page_size = get_page_size(); + return ((reinterpret_cast(buf + len - 1) % page_size) + + padding >= static_cast(page_size)); +} +#endif // SIMDJSON_CPLUSPLUS17 + +inline padded_string_view slice_at(padded_string_view data, char delimiter, + size_t block_size, size_t index) noexcept { + if (block_size == 0 || index > data.size() / block_size) { return {}; } + const size_t raw_begin = index * block_size; + if (raw_begin >= data.size()) { return {}; } + + auto snap = [&](size_t want) -> size_t { + if (want >= data.size()) { return data.size(); } + const void *p = std::memchr(data.data() + want, delimiter, data.size() - want); + return p ? size_t(static_cast(p) - data.data()) + 1 : data.size(); + }; + + const size_t begin = (raw_begin == 0) ? 0 : snap(raw_begin); + const size_t end = snap(raw_begin + block_size); + if (begin >= end) { return {}; } + return padded_string_view(data.data() + begin, end - begin, + data.capacity() - begin); +} } // namespace simdjson @@ -4688,13 +5201,18 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept { #include #include -#ifndef _WIN32 +#if SIMDJSON_HAS_UNISTD_H #include #include #include #include #include #endif +// On Windows, `padded_memory_map` (when it is enabled) depends on types and +// functions declared in . We deliberately do NOT include that +// header here: users of simdjson who want `padded_memory_map` on Windows +// must include themselves *before* including this header. See +// padded_string.h for the detection logic. namespace simdjson { namespace internal { @@ -5064,7 +5582,9 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept { } -#ifndef _WIN32 +#if SIMDJSON_HAS_PADDED_MEMORY_MAP + +#if SIMDJSON_HAS_UNISTD_H simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { int fd = open(filename, O_RDONLY); @@ -5100,7 +5620,132 @@ simdjson_inline padded_memory_map::~padded_memory_map() noexcept { munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); } } +#elif defined(_WIN32) +// Windows zero-copy implementation using placeholder virtual memory. +// +// We use the modern Windows memory APIs (VirtualAlloc2, CreateFileMapping2, +// MapViewOfFile3 -- available since Windows 10 1803) to map the file into a +// contiguous virtual address range that includes at least SIMDJSON_PADDING +// zero bytes after the file content, with no data copies. +// +// Strategy: +// 1. If rounding the file size up to the allocation granularity already +// exceeds file_size + SIMDJSON_PADDING, the OS page zero-fill provides +// the padding and we use a simple MapViewOfFile3 call. +// 2. Otherwise we reserve a contiguous placeholder region via VirtualAlloc2, +// split it at the granularity-aligned file boundary, map the file into +// the first part, and commit zero pages for the second part (padding). +simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { + HANDLE file_handle = ::CreateFileA( + filename, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) { + return; + } + LARGE_INTEGER file_size_li; + if (!::GetFileSizeEx(file_handle, &file_size_li) || file_size_li.QuadPart < 0) { + ::CloseHandle(file_handle); + return; + } +#if SIMDJSON_IS_32BITS + if (static_cast(file_size_li.QuadPart) > + static_cast(SIZE_MAX - simdjson::SIMDJSON_PADDING)) { + ::CloseHandle(file_handle); + return; + } +#endif + size = static_cast(file_size_li.QuadPart); + if (size == 0) { + ::CloseHandle(file_handle); + return; + } + + HANDLE section = ::CreateFileMapping2( + file_handle, NULL, FILE_MAP_READ, PAGE_READONLY, + 0, 0, NULL, NULL, 0); + ::CloseHandle(file_handle); + if (section == NULL) { + return; + } + + SYSTEM_INFO si; + ::GetSystemInfo(&si); + const size_t granularity = static_cast(si.dwAllocationGranularity); + const size_t file_region = (size + granularity - 1) & ~(granularity - 1); + const size_t total_needed = size + simdjson::SIMDJSON_PADDING; + + if (file_region >= total_needed) { + // The zero-fill in the last page already covers the padding. + PVOID view = ::MapViewOfFile3( + section, ::GetCurrentProcess(), NULL, 0, 0, + 0, PAGE_READONLY, NULL, 0); + ::CloseHandle(section); + if (view != NULL) { + data = static_cast(view); + } + return; + } + + // We need extra zero pages beyond the file region. Use the placeholder API + // to get a contiguous virtual address range spanning both the file mapping + // and the zero-filled padding. + const size_t padding_region = + ((total_needed - file_region) + granularity - 1) & ~(granularity - 1); + const size_t reserve_size = file_region + padding_region; + + // Reserve a contiguous placeholder. + PVOID placeholder = ::VirtualAlloc2( + ::GetCurrentProcess(), NULL, reserve_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, NULL, 0); + if (placeholder == NULL) { + ::CloseHandle(section); + return; + } + // Split into two placeholders at the file_region boundary. + if (!::VirtualFree(placeholder, file_region, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { + ::VirtualFree(placeholder, 0, MEM_RELEASE); + ::CloseHandle(section); + return; + } + + // Map the file into the first placeholder. + PVOID file_view = ::MapViewOfFile3( + section, ::GetCurrentProcess(), placeholder, 0, file_region, + MEM_REPLACE_PLACEHOLDER, PAGE_READONLY, NULL, 0); + ::CloseHandle(section); + if (file_view == NULL) { + ::VirtualFree(placeholder, 0, MEM_RELEASE); + ::VirtualFree(static_cast(placeholder) + file_region, + 0, MEM_RELEASE); + return; + } + + // Commit zero pages in the second placeholder (the padding). + void *pad = static_cast(placeholder) + file_region; + PVOID padding_ptr = ::VirtualAlloc2( + ::GetCurrentProcess(), pad, padding_region, + MEM_REPLACE_PLACEHOLDER | MEM_COMMIT, PAGE_READONLY, NULL, 0); + if (padding_ptr == NULL) { + ::UnmapViewOfFile(file_view); + ::VirtualFree(pad, 0, MEM_RELEASE); + return; + } + + data = static_cast(file_view); + padding_view_ = padding_ptr; +} + +simdjson_inline padded_memory_map::~padded_memory_map() noexcept { + if (data == nullptr) { return; } + ::UnmapViewOfFile(data); + if (padding_view_ != nullptr) { + ::VirtualFree(padding_view_, 0, MEM_RELEASE); + } +} +#endif // POSIX or _WIN32 simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { if(!is_valid()) { @@ -5112,7 +5757,8 @@ simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noe simdjson_inline bool padded_memory_map::is_valid() const noexcept { return data != nullptr; } -#endif // _WIN32 + +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP } // namespace simdjson @@ -5222,6 +5868,9 @@ class tape_ref { simdjson_inline tape_ref() noexcept; simdjson_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; inline size_t after_element() const noexcept; + // The reference must point to an element boundary inside the array whose + // opening tag is at array_start, or to that array's closing tag. + inline size_t before_element(size_t array_start) const noexcept; simdjson_inline tape_type tape_ref_type() const noexcept; simdjson_inline uint64_t tape_value() const noexcept; simdjson_inline bool is_double() const noexcept; @@ -5310,6 +5959,48 @@ class array { friend class array; }; + /** + * A forward iterator that visits the array's elements in reverse order. + * Like iterator, it returns element handles by value and does not own the + * document. No allocation or modification of the document is performed. + */ + class reverse_iterator { + public: + using value_type = element; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; + + inline reference operator*() const noexcept; + inline reverse_iterator& operator++() noexcept; + inline reverse_iterator operator++(int) noexcept; + inline bool operator==(const reverse_iterator& other) const noexcept; + inline bool operator!=(const reverse_iterator& other) const noexcept; + + reverse_iterator() noexcept = default; + reverse_iterator(const reverse_iterator&) noexcept = default; + reverse_iterator& operator=(const reverse_iterator&) noexcept = default; + private: + simdjson_inline reverse_iterator(const internal::tape_ref &tape, size_t array_start) noexcept; + internal::tape_ref tape{}; + size_t array_start{}; + friend class array; + }; + + /** + * Return the last array element, or rend() for an empty array. + * Incrementing the returned iterator moves toward the first element. + * A complete traversal takes O(n) time and O(1) additional space, where n + * is the number of immediate elements in the array. + * Finding the last or previous element can take O(n) time in the worst case when + * numeric payloads equal numeric type markers. Increments are amortized O(1) + * over a complete traversal; nested values do not increase this bound. + */ + inline reverse_iterator rbegin() const noexcept; + /** Return the reverse traversal sentinel, before the first element. */ + inline reverse_iterator rend() const noexcept; + /** * Return the first array element. * @@ -5445,6 +6136,8 @@ struct simdjson_result : public internal::simdjson_result_base parse(const char *buf) noexcept = delete; + /** + * Parse a JSON document whose buffer is **not** padded, in place and without + * copying it. + * + * *This feature is currently experimental.* + * + * The standard parse() methods require the input buffer to have at least + * SIMDJSON_PADDING extra readable bytes after the document (or they copy it + * into a padded buffer when realloc_if_needed is true). parse_unpadded() lifts + * that requirement: it parses directly from your buffer of exactly `len` bytes, + * never reading past `buf + len`, and never allocating a full padded copy. + * + * dom::parser parser; + * std::string_view json = get_json(); // no trailing padding needed + * dom::element doc = parser.parse_unpadded(json); + * + * This is the convenient way to use simdjson when you cannot (or do not want + * to) pad your input, e.g. a std::string_view into a larger buffer or a memory + * mapped file whose tail you do not control. It is generally a little slower + * than parsing a padded buffer with parse() (the very end of the document is + * handled with extra care), but it avoids the O(n) copy that + * parse(buf, len, true) performs when realloc_if_needed is true. + * + * The input is read but not modified, and it must remain valid (and the parser + * alive) for as long as you navigate the returned document, exactly like + * parse(buf, len, false). + * + * @param buf The JSON to parse. Only `len` bytes are read; no padding required. + * @param len The length of the JSON. + * @return An element pointing at the root of the document, or an error: + * - MEMALLOC if the parser does not have enough capacity and allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. + */ + inline simdjson_result parse_unpadded(const uint8_t *buf, size_t len) & noexcept; + inline simdjson_result parse_unpadded(const uint8_t *buf, size_t len) && =delete; + /** @overload parse_unpadded(const uint8_t *buf, size_t len) */ + simdjson_inline simdjson_result parse_unpadded(const char *buf, size_t len) & noexcept; + simdjson_inline simdjson_result parse_unpadded(const char *buf, size_t len) && =delete; + /** @overload parse_unpadded(const uint8_t *buf, size_t len) */ + simdjson_inline simdjson_result parse_unpadded(std::string_view s) & noexcept; + simdjson_inline simdjson_result parse_unpadded(std::string_view s) && =delete; + + /** + * Parse a non-padded JSON document into a caller-provided document instance, in + * place and without copying. This is to parse_unpadded() what + * parse_into_document() is to parse(). See parse_unpadded() for the padding and + * lifetime semantics. + * + * *This feature is currently experimental.* + * + * @param doc The document instance where the parsed data will be stored (on success). + * @param buf The JSON to parse. Only `len` bytes are read; no padding required. + * @param len The length of the JSON. + */ + inline simdjson_result parse_into_document_unpadded(document& doc, const uint8_t *buf, size_t len) & noexcept; + inline simdjson_result parse_into_document_unpadded(document& doc, const uint8_t *buf, size_t len) && =delete; + /** * Parse a JSON document into a provide document instance and return a temporary reference to it. * It is similar to the function `parse` except that instead of parsing into the internal @@ -6045,7 +6796,7 @@ class parser { * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. - * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * Defaults to 1MB, which has been a reasonable sweet spot in our tests. * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - MEMALLOC if the parser does not have enough capacity and memory allocation fails * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. @@ -6057,14 +6808,50 @@ class parser { inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const std::string &&s, size_t batch_size) = delete;// unsafe + inline simdjson_result parse_many(const std::string &&s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) = delete;// unsafe /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe + inline simdjson_result parse_many(const padded_string &&s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) = delete;// unsafe + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) + * + * Because padded_string_view guarantees SIMDJSON_PADDING trailing bytes, this + * overload is safe to use with buffers that the caller owns elsewhere (for + * example, a padded_memory_map), with no extra copy. Without this overload, + * passing a padded_string_view would silently bind to the padded_string + * overload via an implicit conversion, allocating and copying the input, and + * -- because that temporary is destroyed at the end of the full-expression -- + * leaving the returned document_stream pointing at freed memory. */ + inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; /** @private We do not want to allow implicit conversion from C string to std::string. */ simdjson_result parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete; + /** + * Parse a stream of JSON documents with explicit format specification. + * + * @param buf The concatenated JSON documents. + * @param len The length of the buffer. + * @param batch_size The batch size to use. + * @param format The stream format. + * @return A stream of documents, or an error. + */ + inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept; + /** @private An rvalue input is destroyed at the end of the full-expression, while + * the returned document_stream only holds a pointer to it: iterating the stream would + * then read freed memory. These deleted overloads also catch a std::string_view + * argument, which would otherwise convert implicitly to a padded_string temporary. */ + inline simdjson_result parse_many(const std::string &&s, size_t batch_size, stream_format format) = delete;// unsafe + /** @private @overload parse_many(const std::string &&s, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const padded_string &&s, size_t batch_size, stream_format format) = delete;// unsafe + /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length * and `max_depth` depth. @@ -6354,8 +7141,9 @@ class document_stream { * * IMPORTANT: this value is only meaningful under the conditions below. It is * computed from stage-1 bookkeeping, and outside these conditions it is not - * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap - * around to a huge value. Check it only when both of the following hold: + * merely imprecise, it is + * arbitrary -- it can exceed size_in_bytes() or wrap around to a huge value. + * Check it only when all of the following hold: * * - you iterated all the way to the end of the stream; * - no document reported an error. Iteration stops at the first failed @@ -6364,6 +7152,9 @@ class document_stream { * If you need to know about a truncated tail outside those conditions, track * it yourself from the last successful document (see iterator::current_index() * and iterator::source()). + * + * An empty input (zero bytes) or an input made only of white space contains + * no document: truncated_bytes() returns zero. */ inline size_t truncated_bytes() const noexcept; /** @@ -6463,12 +7254,14 @@ class document_stream { * @param buf is the raw byte buffer we need to process * @param len is the length of the raw byte buffer in bytes * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) + * @param format is the stream format */ simdjson_inline document_stream( dom::parser &parser, const uint8_t *buf, size_t len, - size_t batch_size + size_t batch_size, + stream_format format = stream_format::whitespace_delimited ) noexcept; /** @@ -6518,6 +7311,8 @@ class document_stream { const uint8_t *buf; size_t len; size_t batch_size; + /** The stream format. */ + stream_format format; /** The error (or lack thereof) from the current document. */ error_code error; size_t batch_start{0}; @@ -6605,6 +7400,8 @@ enum class element_type { STRING = '"', ///< std::string_view BOOL = 't', ///< bool NULL_VALUE = 'n', ///< null + /// The BIGINT type is for integers that do not fit in 64 bits. It is only present + // if you set parser.number_as_string(true). BIGINT = 'Z' ///< std::string_view: big integer stored as raw digit string }; @@ -6673,6 +7470,20 @@ class element { * Returns INCORRECT_TYPE if the JSON element is not a string. */ inline simdjson_result get_string() const noexcept; + + #if SIMDJSON_SUPPORTS_CHAR8_T + /** + * Cast this element to a C++20 UTF-8 string. + * + * The string is guaranteed to be valid UTF-8. + * + * @returns A std::u8string_view. The string is stored in the parser and will be invalidated the next time it + * parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_u8string() const noexcept; + #endif + /** * Cast this element to a signed integer. * @@ -6778,7 +7589,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -6793,7 +7604,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -6823,7 +7634,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -6842,7 +7653,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -7126,6 +7937,9 @@ struct simdjson_result : public internal::simdjson_result_base get_c_str() const noexcept; simdjson_inline simdjson_result get_string_length() const noexcept; simdjson_inline simdjson_result get_string() const noexcept; + #if SIMDJSON_SUPPORTS_CHAR8_T + simdjson_inline simdjson_result get_u8string() const noexcept; + #endif simdjson_inline simdjson_result get_int64() const noexcept; simdjson_inline simdjson_result get_uint64() const noexcept; simdjson_inline simdjson_result get_double() const noexcept; @@ -7825,6 +8639,24 @@ template std::string prettify(simdjson_result x) { namespace simdjson { +/** Specifies where commas should be in table-formatted elements. */ +enum class table_comma_placement { + /** Commas come right after the value */ + before_padding, + /** Commas come after the column padding, so they line up in their own column. */ + after_padding, + /** Commas come right after the value, except for columns of numbers */ + before_padding_except_numbers, +}; + +/** Options for how lists or columns of numbers should be aligned */ +enum class number_list_alignment { + /** Left-aligns numbers */ + left, + /** Right-aligns numbers */ + right, +}; + /** * Configuration options for FracturedJson formatting. * @@ -7839,12 +8671,6 @@ struct fractured_json_options { */ size_t max_total_line_length = 120; - /** - * Maximum length for inlined elements (default: 80). - * Simple arrays/objects shorter than this may be rendered inline. - */ - size_t max_inline_length = 80; - /** * Maximum nesting depth for inline rendering (default: 2). * Elements with complexity exceeding this will be expanded. @@ -7853,11 +8679,11 @@ struct fractured_json_options { size_t max_inline_complexity = 2; /** - * Maximum complexity for compact array formatting (default: 1). + * Maximum complexity for compact array formatting (default: 2). * Arrays with elements of this complexity or less may have multiple * items per line. */ - size_t max_compact_array_complexity = 1; + size_t max_compact_array_complexity = 2; /** * Number of spaces per indentation level (default: 4). @@ -7865,23 +8691,26 @@ struct fractured_json_options { size_t indent_spaces = 4; /** - * Enable tabular formatting for arrays of similar objects (default: true). - * When enabled, arrays of objects with identical keys are formatted - * as aligned tables. + * Forces elements close to the root to always fully expand, regardless of other settings. + * (default: -1). -1 = none; 0 = root node only; 1 = root node and its children; etc. */ - bool enable_table_format = true; + int always_expand_depth = -1; /** - * Minimum number of rows to trigger table mode (default: 3). + * Enable tabular formatting for arrays of similar objects or arrays + * (default: true). When enabled, the rows of such an array are written one + * per line with their columns aligned. Rows need not have identical keys: + * columns are ordered by the first occurrence of each key, and a row + * missing a key gets blank space in that column. */ - size_t min_table_rows = 3; + bool enable_table_format = true; /** - * Similarity threshold for table detection (default: 0.8). - * Objects must share at least this fraction of keys to be formatted - * as a table. + * Maximum complexity of each row of a table (default: 2). + * 0 = rows may only be scalars (a single column); 1 = rows may be flat + * arrays/objects; higher values allow deeper nesting. */ - double table_similarity_threshold = 0.8; + size_t max_table_row_complexity = 2; /** * Enable compact multiline arrays (default: true). @@ -7891,16 +8720,26 @@ struct fractured_json_options { bool enable_compact_multiline = true; /** - * Maximum array items per line in compact mode (default: 10). + * Minimum number of items per line for an array to be formatted as a + * compact multiline array (default: 3). */ - size_t max_items_per_line = 10; + size_t min_compact_array_row_items = 3; /** - * Add space inside brackets for simple containers (default: true). - * When true: { "key": "value" } - * When false: {"key": "value"} + * Add space inside brackets for containers that hold only scalar values + * (default: false). When true: { "key": "value" }. When false: + * {"key": "value"}. + * @see nested_bracket_padding */ - bool simple_bracket_padding = true; + bool simple_bracket_padding = false; + + /** + * Add space inside brackets for containers that hold at least one + * nested array/object (default: true). When true: { "a": [1, 2] }. + * When false: {"a": [1, 2]}. + * @see simple_bracket_padding + */ + bool nested_bracket_padding = true; /** * Add space after colons (default: true). @@ -7915,6 +8754,18 @@ struct fractured_json_options { * When false: [1,2,3] */ bool comma_padding = true; + + /** + * Placement of commas relative to column padding in table-formatted rows + * and compact multiline arrays (default: before_padding_except_numbers). + */ + table_comma_placement comma_placement = table_comma_placement::before_padding_except_numbers; + + /** + * Controls alignment of numbers in table columns or compact multiline arrays + * (default: left). Numbers are always written exactly as in the input. + */ + number_list_alignment number_alignment = number_list_alignment::left; }; /** @@ -7995,12 +8846,55 @@ inline std::string fractured_json_string(std::string_view json_str, #ifndef SIMDJSON_JSONPATHUTIL_H #define SIMDJSON_JSONPATHUTIL_H +/* skipped duplicate #include "simdjson/error.h" */ #include /* skipped duplicate #include "simdjson/common_defs.h" */ +#include #include namespace simdjson { +namespace internal { +/** + * Parses the next JSON Pointer array index token. + * + * The caller passes a pointer fragment with no leading '/', such as "123/foo". + * On success, array_index receives the parsed index and token_length receives + * the number of bytes consumed before the next '/' or the end of the fragment. + */ +simdjson_inline error_code parse_json_pointer_array_index(std::string_view json_pointer, + size_t &array_index, + size_t &token_length) noexcept { + array_index = 0; + token_length = 0; + + for (; token_length < json_pointer.length() && json_pointer[token_length] != '/'; + token_length++) { + uint8_t digit = uint8_t(json_pointer[token_length] - '0'); + // Check for non-digit in array index. If it's there, we're trying to get a field in an object. + if (digit > 9) { + return INCORRECT_TYPE; + } + // 0 followed by other digits is invalid. + if (token_length > 0 && json_pointer[0] == '0') { + return INVALID_JSON_POINTER; + } + if (array_index > + (((std::numeric_limits::max)() - digit) / 10)) { + return INDEX_OUT_OF_BOUNDS; + } + array_index = array_index * 10 + digit; + } + + // Empty string is invalid; so is a "/" with no digits before it. + if (token_length == 0) { + return INVALID_JSON_POINTER; + } + + return SUCCESS; +} +} // namespace internal + /** * Converts JSONPath to JSON Pointer. * @param json_path The JSONPath string to be converted. @@ -8220,6 +9114,72 @@ inline size_t tape_ref::after_element() const noexcept { simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { return static_cast(doc->tape[json_index] >> 56); } +simdjson_inline size_t tape_ref::before_element(size_t array_start) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(usable()); + SIMDJSON_DEVELOPMENT_ASSERT(json_index > array_start); + tape_ref previous(doc, json_index - 1); + if (previous.json_index == array_start) { return array_start; } + // An exact numeric marker cannot end an element unless it is itself the + // payload of a number. In that case its header is immediately before it. + if (previous.is_int64() || previous.is_uint64() || previous.is_double()) { + return previous.json_index - 1; + } + tape_ref probe(doc, previous.json_index - 1); + + // Validate both container links before examining its contents. A candidate + // opening tag preceded by an even run of numeric markers is a real tag, + // not a numeric payload. Scan both candidate boundaries together so that a + // forged opening tag inside a nested value cannot cause an unbounded detour. + // For a real container, the opening probe visits preceding siblings. For a + // numeric payload, the other probe does. Stopping at the shorter run bounds + // the work by the array's immediate elements, rather than nested contents. + const auto type = previous.tape_ref_type(); + if (type == tape_type::END_ARRAY || type == tape_type::END_OBJECT) { + const size_t start = previous.matching_brace_index(); + if (start > array_start && start < previous.json_index) { + tape_ref opening(doc, start); + const auto expected = type == tape_type::END_ARRAY + ? tape_type::START_ARRAY : tape_type::START_OBJECT; + if (opening.tape_ref_type() == expected && + opening.matching_brace_index() == previous.json_index + 1) { + tape_ref before_opening(doc, start - 1); + while ((before_opening.is_int64() || before_opening.is_uint64() || + before_opening.is_double()) && + (probe.is_int64() || probe.is_uint64() || probe.is_double())) { + --before_opening.json_index; + --probe.json_index; + } + if (!before_opening.is_int64() && !before_opening.is_uint64() && + !before_opening.is_double() && + (start - before_opening.json_index) % 2 == 1) { + return start; + } + } + } + } + + // Numeric payloads can have ANY bit pattern, including another type's tag. + // A run of exact numeric markers starts with a header, then alternates + // between payload and header. An odd run before this word makes it a payload. + // Subsequent reverse increments through exact-marker payloads take the + // constant-time numeric-payload branch above. + while (probe.is_int64() || probe.is_uint64() || probe.is_double()) { + --probe.json_index; + } + if ((previous.json_index - probe.json_index) % 2 == 0) { + return previous.json_index - 1; + } + + // Once distinguished from numeric payloads, closing container tags link + // directly back to their opening tags. + switch (previous.tape_ref_type()) { + case tape_type::END_ARRAY: + case tape_type::END_OBJECT: + return previous.matching_brace_index(); + default: + return previous.json_index; + } +} simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { return doc->tape[json_index] & internal::JSON_VALUE_MASK; } @@ -8295,6 +9255,14 @@ inline size_t simdjson_result::size() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.size(); } +inline dom::array::reverse_iterator simdjson_result::rbegin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.rbegin(); +} +inline dom::array::reverse_iterator simdjson_result::rend() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.rend(); +} #endif // SIMDJSON_EXCEPTIONS @@ -8304,6 +9272,7 @@ inline simdjson_result simdjson_result::at_pointer(std } inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { + if (error()) { return error(); } auto json_pointer = json_path_to_pointer_conversion(json_path); if (json_pointer == "-1") { return INVALID_JSON_POINTER; } return at_pointer(json_pointer); @@ -8344,6 +9313,15 @@ inline size_t array::size() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.scope_count(); } +inline array::reverse_iterator array::rbegin() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); + const internal::tape_ref end_tape(tape.doc, tape.matching_brace_index() - 1); + return reverse_iterator(internal::tape_ref(tape.doc, end_tape.before_element(tape.json_index)), tape.json_index); +} +inline array::reverse_iterator array::rend() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); + return reverse_iterator(tape, tape.json_index); +} inline size_t array::number_of_slots() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.matching_brace_index() - tape.json_index; @@ -8360,21 +9338,9 @@ inline simdjson_result array::at_pointer(std::string_view json_pointer) // We don't support this, because we're returning a real element, not a position. if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } - // Read the array index size_t array_index = 0; size_t i; - for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { - uint8_t digit = uint8_t(json_pointer[i] - '0'); - // Check for non-digit in array index. If it's there, we're trying to get a field in an object - if (digit > 9) { return INCORRECT_TYPE; } - array_index = array_index*10 + digit; - } - - // 0 followed by other digits is invalid - if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" - - // Empty string is invalid; so is a "/" with no digits before it - if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" + SIMDJSON_TRY(internal::parse_json_pointer_array_index(json_pointer, array_index, i)); // Get the child auto child = array(tape).at(array_index); @@ -8409,7 +9375,6 @@ inline void array::process_json_path_of_child_elements(std::vector::ite if(error) { continue; } - accumulator.reserve(accumulator.size() + child_result.size()); accumulator.insert(accumulator.end(), std::make_move_iterator(child_result.begin()), std::make_move_iterator(child_result.end())); @@ -8505,6 +9470,30 @@ inline array::operator element() const noexcept { return element(tape); } +// +// array::reverse_iterator inline implementation +// +simdjson_inline array::reverse_iterator::reverse_iterator(const internal::tape_ref &_tape, size_t _array_start) noexcept + : tape{_tape}, array_start{_array_start} { } +inline element array::reverse_iterator::operator*() const noexcept { + return element(tape); +} +inline array::reverse_iterator& array::reverse_iterator::operator++() noexcept { + tape.json_index = tape.before_element(array_start); + return *this; +} +inline array::reverse_iterator array::reverse_iterator::operator++(int) noexcept { + reverse_iterator out = *this; + ++*this; + return out; +} +inline bool array::reverse_iterator::operator==(const reverse_iterator& other) const noexcept { + return tape.doc == other.tape.doc && tape.json_index == other.tape.json_index; +} +inline bool array::reverse_iterator::operator!=(const reverse_iterator& other) const noexcept { + return !(*this == other); +} + // // array::iterator inline implementation // @@ -8596,6 +9585,7 @@ inline simdjson_result simdjson_result::at_pointer(st return first.at_pointer(json_pointer); } inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { + if (error()) { return error(); } auto json_pointer = json_path_to_pointer_conversion(json_path); if (json_pointer == "-1") { return INVALID_JSON_POINTER; } return at_pointer(json_pointer); @@ -8725,7 +9715,6 @@ inline void object::process_json_path_of_child_elements(std::vector::it if(error) { continue; } - accumulator.reserve(accumulator.size() + child_result.size()); accumulator.insert(accumulator.end(), std::make_move_iterator(child_result.begin()), std::make_move_iterator(child_result.end())); @@ -9003,6 +9992,12 @@ simdjson_inline simdjson_result simdjson_result: if (error()) { return error(); } return first.get_string(); } +#if SIMDJSON_SUPPORTS_CHAR8_T +simdjson_inline simdjson_result simdjson_result::get_u8string() const noexcept { + if (error()) { return error(); } + return first.get_u8string(); +} +#endif simdjson_inline simdjson_result simdjson_result::get_int64() const noexcept { if (error()) { return error(); } return first.get_int64(); @@ -9069,6 +10064,7 @@ simdjson_inline simdjson_result simdjson_result::at_ return first.at_pointer(json_pointer); } simdjson_inline simdjson_result simdjson_result::at_path(const std::string_view json_path) const noexcept { + if (error()) { return error(); } auto json_pointer = json_path_to_pointer_conversion(json_path); if (json_pointer == "-1") { return INVALID_JSON_POINTER; } return at_pointer(json_pointer); @@ -9201,6 +10197,13 @@ inline simdjson_result element::get_string() const noexcept { return INCORRECT_TYPE; } } +#if SIMDJSON_SUPPORTS_CHAR8_T +inline simdjson_result element::get_u8string() const noexcept { + std::string_view v; + SIMDJSON_TRY(get_string().get(v)); + return internal::as_u8string_view(v); +} +#endif inline simdjson_result element::get_uint64() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken @@ -9296,6 +10299,9 @@ template<> inline simdjson_result element::get() const noexcept { template<> inline simdjson_result element::get() const noexcept { return get_object(); } template<> inline simdjson_result element::get() const noexcept { return get_c_str(); } template<> inline simdjson_result element::get() const noexcept { return get_string(); } +#if SIMDJSON_SUPPORTS_CHAR8_T +template<> inline simdjson_result element::get() const noexcept { return get_u8string(); } +#endif template<> inline simdjson_result element::get() const noexcept { return get_int64(); } template<> inline simdjson_result element::get() const noexcept { return get_uint64(); } template<> inline simdjson_result element::get() const noexcept { return get_double(); } @@ -9628,6 +10634,29 @@ inline simdjson_result parser::parse_into_document(document& provided_d return provided_doc.root(); } +inline simdjson_result parser::parse_into_document_unpadded(document& provided_doc, const uint8_t *buf, size_t len) & noexcept { + // Like parse_into_document with realloc_if_needed=false (no copy, parse in + // place), but we tell the implementation the buffer is not padded so stage 2 + // avoids reading past buf+len: string unescaping is bounded, near-the-end + // numbers are parsed from a padded copy, and atoms use length-aware + // validators (see tape_builder). Stage 1 is already safe for unpadded input. + error_code _error = ensure_capacity(provided_doc, len); + if (_error) { return _error; } + + if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + implementation->_number_as_string = _number_as_string; + implementation->_unpadded = true; + _error = implementation->parse(buf, len, provided_doc); + implementation->_unpadded = false; // restore so later padded parses use the fast path + + if (_error) { return _error; } + + return provided_doc.root(); +} + simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char *buf, size_t len, bool realloc_if_needed) & noexcept { return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); } @@ -9656,13 +10685,18 @@ simdjson_inline simdjson_result parser::parse(const padded_string_view return parse(v.data(), v.length(), false); } +inline simdjson_result parser::parse_unpadded(const uint8_t *buf, size_t len) & noexcept { + return parse_into_document_unpadded(doc, buf, len); +} +simdjson_inline simdjson_result parser::parse_unpadded(const char *buf, size_t len) & noexcept { + return parse_unpadded(reinterpret_cast(buf), len); +} +simdjson_inline simdjson_result parser::parse_unpadded(std::string_view s) & noexcept { + return parse_unpadded(reinterpret_cast(s.data()), s.size()); +} + inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { - if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - return document_stream(*this, buf, len, batch_size); + return parse_many(buf, len, batch_size, stream_format::whitespace_delimited); } inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { return parse_many(reinterpret_cast(buf), len, batch_size); @@ -9673,6 +10707,48 @@ inline simdjson_result parser::parse_many(const std::string &s, inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } +inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size) noexcept { + return parse_many(v.data(), v.length(), batch_size); +} + +inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + if (format == stream_format::comma_delimited_array) { + // Strip leading JSON whitespace. + while (len > 0 && (buf[0] == ' ' || buf[0] == '\t' || buf[0] == '\n' || buf[0] == '\r')) { + buf++; len--; + } + // Expect the opening '['. + if (len == 0 || buf[0] != '[') { return TAPE_ERROR; } + buf++; len--; + // Strip trailing JSON whitespace. + while (len > 0 && (buf[len-1] == ' ' || buf[len-1] == '\t' || buf[len-1] == '\n' || buf[len-1] == '\r')) { + len--; + } + // Expect the closing ']'. + if (len == 0 || buf[len-1] != ']') { return TAPE_ERROR; } + len--; + // Fall through to comma_delimited over the array contents. + format = stream_format::comma_delimited; + } + return document_stream(*this, buf, len, batch_size, format); +} +inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept { + return parse_many(reinterpret_cast(buf), len, batch_size, format); +} +inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept { + return parse_many(s.data(), s.length(), batch_size, format); +} +inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept { + return parse_many(s.data(), s.length(), batch_size, format); +} +inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept { + return parse_many(v.data(), v.length(), batch_size, format); +} simdjson_inline size_t parser::capacity() const noexcept { return implementation ? implementation->capacity() : 0; @@ -9830,12 +10906,14 @@ simdjson_inline document_stream::document_stream( dom::parser &_parser, const uint8_t *_buf, size_t _len, - size_t _batch_size + size_t _batch_size, + stream_format _format ) noexcept : parser{&_parser}, buf{_buf}, len{_len}, batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size}, + format{_format}, error{SUCCESS} #ifdef SIMDJSON_THREADS_ENABLED , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change @@ -9853,6 +10931,7 @@ simdjson_inline document_stream::document_stream() noexcept buf{nullptr}, len{0}, batch_size{0}, + format{stream_format::whitespace_delimited}, error{UNINITIALIZED} #ifdef SIMDJSON_THREADS_ENABLED , use_thread(false) @@ -9929,6 +11008,7 @@ inline void document_stream::start() noexcept { if (error) { return; } error = parser->ensure_capacity(batch_size); if (error) { return; } + parser->implementation->_number_as_string = parser->number_as_string(); // Always run the first stage 1 parse immediately batch_start = 0; error = run_stage1(*parser, batch_start); @@ -9968,7 +11048,40 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc } else { size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index]; size_t svlen = next_doc_index - current_index(); - while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) { + // When the scalar is followed by a truncated document, the structural + // indexes of that document were dropped and next_doc_index is the end of + // the input, so we bound the scalar by scanning the token itself. + size_t token_len = 0; + if (*start == '"') { + token_len = 1; + while (token_len < svlen) { + char c = start[token_len++]; + if (c == '\\') { + token_len++; + } else if (c == '"') { + break; + } + } + } else { + while (token_len < svlen) { + char c = start[token_len]; + if (std::isspace(static_cast(c)) || c == ',' || c == '{' || c == '[' || c == '\0' || static_cast(c) == 0x1E) { + break; + } + token_len++; + } + } + if (token_len > 0 && token_len < svlen) { + svlen = token_len; + } + // Trim trailing whitespace, NUL, and RS (0x1E). In RFC 7464 json_sequence + // mode the scanner classifies RS as a scalar character, so an RS-prefixed + // scalar document (number/true/false/null/string) has no closing structural + // index and the slice runs all the way up to the next document's RS. RS + // cannot legally appear in a JSON value at the source level (control + // characters in strings must be escaped as \u001E), so stripping it is + // safe in every stream_format. + while(svlen > 1 && (std::isspace(static_cast(start[svlen-1])) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E || (stream->format == stream_format::comma_delimited && start[svlen-1] == ','))) { svlen--; } return std::string_view(start, svlen); @@ -10008,6 +11121,9 @@ inline size_t document_stream::size_in_bytes() const noexcept { } inline size_t document_stream::truncated_bytes() const noexcept { + // Stage 1 returns EMPTY on zero-length input before it writes the index + // sentinels read below, so they would still hold a previous stream's values. + if (len == 0) { return 0; } if(error == CAPACITY) { return len - batch_start; } return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; } @@ -10018,10 +11134,35 @@ inline size_t document_stream::next_batch_start() const noexcept { inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { size_t remaining = len - _batch_start; + stage1_mode mode; if (remaining <= batch_size) { - return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final); + // Final batch + switch (format) { + case stream_format::json_sequence: + mode = stage1_mode::json_sequence_final; + break; + case stream_format::comma_delimited: + mode = stage1_mode::comma_delimited_final; + break; + default: + mode = stage1_mode::streaming_final; + break; + } + return p.implementation->stage1(&buf[_batch_start], remaining, mode); } else { - return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial); + // Partial batch + switch (format) { + case stream_format::json_sequence: + mode = stage1_mode::json_sequence_partial; + break; + case stream_format::comma_delimited: + mode = stage1_mode::comma_delimited_partial; + break; + default: + mode = stage1_mode::streaming_partial; + break; + } + return p.implementation->stage1(&buf[_batch_start], batch_size, mode); } } @@ -10194,16 +11335,25 @@ inline error_code document::allocate(size_t capacity) noexcept { allocated_capacity = 0; return SUCCESS; } + if (capacity > SIMDJSON_MAXSIZE_BYTES) { + return CAPACITY; + } // a pathological input like "[[[[..." would generate capacity tape elements, so // need a capacity of at least capacity + 1, but it is also possible to do // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6" //where capacity + 1 tape elements are // generated, see issue https://github.com/simdjson/simdjson/issues/345 + if(capacity + 3 < capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64); // a document with only zero-length strings... could have capacity/3 string // and we would need capacity/3 * 5 bytes on the string buffer - size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * capacity / 3 + SIMDJSON_PADDING, 64); + if(5 * (capacity / 3) + SIMDJSON_PADDING < SIMDJSON_PADDING) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * (capacity / 3) + SIMDJSON_PADDING, 64); string_buf.reset( new (std::nothrow) uint8_t[string_capacity]); tape.reset(new (std::nothrow) uint64_t[tape_capacity]); if(!(string_buf && tape)) { @@ -10347,6 +11497,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { /* skipped duplicate #include "simdjson/dom/object-inl.h" */ /* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ +#include #include namespace simdjson { @@ -10517,11 +11668,29 @@ simdjson_inline void base_formatter::number(int64_t x) { template simdjson_inline void base_formatter::number(double x) { - char number_buffer[24]; +#if SIMDJSON_ENABLE_NAN_INF + if (simdjson_unlikely(!std::isfinite(x))) { + if (std::isnan(x)) { + char const *s = "NaN"; + chars(s, s + 3); + } else { + if (x < 0) { + one_char('-'); + } + char const *s = "Infinity"; + chars(s, s + 8); + } + return; + } +#endif + + // Must be to_chars_buffer_size (40): only ~24 chars are emitted, but + // to_chars over-writes with fixed-size 16/17-byte copies for inlining. + char number_buffer[simdjson::internal::to_chars_buffer_size]; // Currently, passing the nullptr to the second argument is // safe because our implementation does not check the second // argument. - char *newp = internal::to_chars(number_buffer, nullptr, x); + char *newp = simdjson::internal::to_chars(number_buffer, nullptr, x); chars(number_buffer, newp); } @@ -10797,7 +11966,7 @@ inline void string_builder::append(simdjson::dom::element value) { format.string(iter.get_string_view()); break; case tape_type::BIGINT: { - // Big integer stored as string — output raw digits (no quotes) + // Big integer stored as string -- output raw digits (no quotes) auto sv = iter.get_string_view(); format.chars(sv.data(), sv.data() + sv.size()); break; @@ -10935,7 +12104,7 @@ simdjson_inline std::string_view string_builder::str() const { #include #include #include -#include +#include namespace simdjson { namespace internal { @@ -10944,12 +12113,50 @@ namespace internal { * Layout mode for fractured JSON formatting. */ enum class layout_mode { - INLINE, // Single line: [1, 2, 3] or {"a": 1} - COMPACT_MULTILINE, // Multiple items per line with breaks - TABLE, // Tabular format for arrays of similar objects - EXPANDED // Traditional multi-line with indentation + single_line, // Single line: [1, 2, 3] or {"a": 1} + compact_multiline, // Multiple items per line with breaks + table, // Tabular format for arrays of similar objects + expanded // Traditional multi-line with indentation }; +/** Kind of value found in a table column across all rows that have one. */ +enum class table_column_type { + unknown, + simple, // string, bool, or null + number, + array, + object, + mixed // rows disagree on kind +}; + +/** Column of a table-formatted array.*/ +struct table_column { + /** Column name for object rows (may be the empty string). */ + std::string key{}; + /** True for object rows (the column has a key), false for array rows. */ + bool has_key = false; + /** Rendered length of key */ + size_t key_width = 0; + table_column_type type = table_column_type::unknown; + /** Widest rendered value in this column (when fully expanding all children) */ + size_t width = 0; + /** Widest plain value in this column. */ + size_t plain_width = 0; + /** subcolumns (only populated if every child is an array or every child is an object) */ + std::vector children{}; +}; + +/** Whether rows from this column list use nested vs. simple bracket padding + * One shared decision, since picking it per-row would misalign width-aligned rows. */ +inline bool table_row_is_nested(const std::vector& columns) { + for (const table_column& col : columns) { + if (col.type == table_column_type::object || col.type == table_column_type::array) { + return true; + } + } + return false; +} + /** * Metrics computed for a JSON element during structure analysis. * These metrics drive layout decisions and contain child metrics for recursive formatting. @@ -10970,16 +12177,32 @@ struct element_metrics { /** Is this an array where all elements have similar structure? */ bool is_uniform_array = false; - /** For uniform arrays of objects: the common keys */ - std::vector common_keys{}; - - /** Recommended layout mode based on analysis */ - layout_mode recommended_layout = layout_mode::EXPANDED; + /** for uniform arrays: this array's columns for alignment */ + std::vector table_columns{}; + /** Widest table row after pruning recursive columns that don't fit into line budget */ + size_t table_row_width = 0; + /** Widest table row without pruning recursive columns that don't fit into line budget */ + size_t table_row_width_full = 0; /** Child metrics for arrays and objects (in order of iteration) */ std::vector children{}; + + /** For scalar uniform arrays (table_columns empty): the rows' common type. */ + table_column_type scalar_column_type = table_column_type::unknown; }; +/** children[idx], or a default-constructed element_metrics if idx is out of range */ +inline const element_metrics& child_metrics_at(const std::vector& children, size_t idx) { + static const element_metrics empty{}; + return idx < children.size() ? children[idx] : empty; +} + +/** *ptr, or a default-constructed element_metrics if ptr is null. */ +inline const element_metrics& child_metrics_at(const element_metrics* ptr) { + static const element_metrics empty{}; + return ptr ? *ptr : empty; +} + /** * Analyzes JSON structure to compute metrics for formatting decisions. * @@ -11040,20 +12263,27 @@ class structure_analyzer { element_metrics analyze_object(const dom::object& obj, const fractured_json_options& opts); + /** Decide layout at the given render depth. Kept out of analysis since + * the same metrics can render inline or expanded at different depths. */ + static layout_mode decide_layout(const element_metrics& metrics, + size_t depth, + const fractured_json_options& opts, + bool has_trailing_comma = false); + private: const fractured_json_options* current_opts_ = nullptr; /** Recursive analysis implementation */ - element_metrics analyze_element(const dom::element& elem, size_t depth); + element_metrics analyze_element(const dom::element& elem, size_t depth) const; /** Analyze scalar values (strings, numbers, booleans, null) */ - element_metrics analyze_scalar(const dom::element& elem); + element_metrics analyze_scalar(const dom::element& elem) const; /** Analyze an array element */ - element_metrics analyze_array(const dom::array& arr, size_t depth); + element_metrics analyze_array(const dom::array& arr, size_t depth) const; /** Analyze an object element */ - element_metrics analyze_object(const dom::object& obj, size_t depth); + element_metrics analyze_object(const dom::object& obj, size_t depth) const; /** Estimate inline length for a string (including quotes and escaping) */ size_t estimate_string_length(std::string_view s) const; @@ -11064,27 +12294,35 @@ class structure_analyzer { size_t estimate_number_length(uint64_t u) const; /** - * Check if an array contains uniform objects suitable for table formatting. + * Check if an array contains uniformly-shaped rows suitable for table + * formatting, filling in corresponding metrics * @param arr The array to check - * @param common_keys Output: keys common to all objects - * @return true if the array is suitable for table formatting + * @param metrics The array's metrics, with children already filled in + * @param depth The array's depth */ - bool check_array_uniformity(const dom::array& arr, - std::vector& common_keys) const; + bool check_array_uniformity(const dom::array& arr, element_metrics& metrics, size_t depth) const; - /** - * Compute similarity between two objects. - * @return Fraction of keys that are common (0.0 to 1.0) - */ - double compute_object_similarity(const dom::object& a, - const dom::object& b) const; + static table_column_type classify_table_value(dom::element_type type); - /** - * Decide the recommended layout mode based on metrics and options. - */ - layout_mode decide_layout(const element_metrics& metrics, - size_t depth, - size_t available_width) const; + /** Find common type across a set of sibling values and the widest of their rendered lengths */ + static void classify_and_measure(const std::vector>& values, + table_column_type& common, size_t& max_width); + + /** Recursively build table columns for an array */ + void build_table_columns(const std::vector>& values, + std::vector& out_columns) const; + + /** Rendered width of a row, assuming each column's current */ + size_t compute_columns_width(const std::vector& columns) const; + + /** Height of a column's recursion (0 = leaf). */ + static size_t column_height(const table_column& column); + + /** Flatten deepest columns of the table */ + static bool flatten_deepest_columns(std::vector& columns); + + /** Bottom-up refresh of column widths after flatten_deepest_columns. */ + void recompute_column_widths(std::vector& columns) const; }; } // namespace internal @@ -11130,56 +12368,35 @@ class fractured_formatter : public base_formatter { /** Get the current layout mode */ layout_mode get_layout_mode() const; - /** Set current depth for formatting decisions */ - void set_depth(size_t depth); - - /** Get current depth */ - size_t get_depth() const; - /** Track current line length for compact multiline decisions */ void track_line_length(size_t chars); - /** Reset line length (after newline) */ - void reset_line_length(); - - /** Get current line length */ - size_t get_line_length() const; - /** Check if we should break to a new line in compact mode */ bool should_break_line(size_t upcoming_length) const; /** Get the options */ const fractured_json_options& options() const; - // Table formatting support - /** Begin a table row */ - void begin_table_row(); - - /** End a table row */ - void end_table_row(); - - /** Set column widths for table alignment */ - void set_column_widths(const std::vector& widths); - - /** Get current column index in table mode */ - size_t get_column_index() const; - - /** Advance to next column */ - void next_column(); - - /** Add padding to align with column width */ - void align_to_column_width(size_t actual_width); - private: fractured_json_options options_; - layout_mode current_layout_ = layout_mode::EXPANDED; - size_t current_depth_ = 0; + layout_mode current_layout_ = layout_mode::expanded; size_t current_line_length_ = 0; +}; - // Table state - bool in_table_mode_ = false; - std::vector column_widths_; - size_t current_column_ = 0; +/** RAII helper forcing single line layout and restoring previous mode on exit */ +class scoped_single_line_mode { +public: + explicit scoped_single_line_mode(fractured_formatter& format) + : format_(format), prev_(format.get_layout_mode()) { + format_.set_layout_mode(layout_mode::single_line); + } + ~scoped_single_line_mode() { format_.set_layout_mode(prev_); } + scoped_single_line_mode(const scoped_single_line_mode&) = delete; + scoped_single_line_mode& operator=(const scoped_single_line_mode&) = delete; + +private: + fractured_formatter& format_; + layout_mode prev_; }; /** @@ -11214,10 +12431,12 @@ class fractured_string_builder { fractured_json_options options_; /** Format an element using pre-computed metrics */ - void format_element(const dom::element& elem, const element_metrics& metrics, size_t depth); + void format_element(const dom::element& elem, const element_metrics& metrics, size_t depth, + bool has_trailing_comma = false); /** Format an array with the appropriate layout */ - void format_array(const dom::array& arr, const element_metrics& metrics, size_t depth); + void format_array(const dom::array& arr, const element_metrics& metrics, size_t depth, + bool has_trailing_comma = false); /** Format an array inline: [1, 2, 3] */ void format_array_inline(const dom::array& arr, const element_metrics& metrics); @@ -11225,14 +12444,51 @@ class fractured_string_builder { /** Format an array with compact multiline: multiple items per line */ void format_array_compact_multiline(const dom::array& arr, const element_metrics& metrics, size_t depth); + /** Like format_array_compact_multiline, but rows are cross-row aligned + * and packed using a fixed per-row slot width. */ + void format_array_compact_multiline_aligned(const dom::array& arr, const element_metrics& metrics, size_t depth); + /** Format an array as a table */ void format_array_as_table(const dom::array& arr, const element_metrics& metrics, size_t depth); + /** Write one object row's columns */ + void format_table_object_row(const dom::object& obj, const element_metrics& row_metrics, + const std::vector& columns, size_t depth); + + /** Write one array row's columns */ + void format_table_array_row(const dom::array& arr, const element_metrics& row_metrics, + const std::vector& columns, size_t depth); + + /** Dispatches to format_table_object_row/format_table_array_row based on elem's type. */ + void format_table_row(const dom::element& elem, const element_metrics& row_metrics, + const std::vector& columns, size_t depth); + + /** Row for a uniform scalar array: writes elem inline, then pads to width so every row lines up. */ + void format_table_scalar_row(const dom::element& elem, const element_metrics& row_metrics, + size_t width, size_t depth, table_column_type column_type); + + /** Shared per-column writer: recurses if the column has children, + * otherwise writes a plain padded value or blank. */ + void format_table_row_columns(const std::vector& columns, + const std::vector& found, + const std::vector& values, + const std::vector& value_metrics, + size_t depth); + + /** Writes a single aligned leaf value */ + void format_table_leaf_value(const dom::element& elem, const element_metrics& vm, size_t width, + table_column_type column_type, bool needs_comma, + bool add_comma_space, size_t depth); + + /** Whether, for a column of the given type, the comma goes right after the value */ + bool comma_goes_before_padding(table_column_type column_type) const; + /** Format an array expanded: one item per line */ void format_array_expanded(const dom::array& arr, const element_metrics& metrics, size_t depth); /** Format an object with the appropriate layout */ - void format_object(const dom::object& obj, const element_metrics& metrics, size_t depth); + void format_object(const dom::object& obj, const element_metrics& metrics, size_t depth, + bool has_trailing_comma = false); /** Format an object inline: {"a": 1, "b": 2} */ void format_object_inline(const dom::object& obj, const element_metrics& metrics); @@ -11243,12 +12499,8 @@ class fractured_string_builder { /** Format a scalar value */ void format_scalar(const dom::element& elem); - /** Calculate column widths for table formatting */ - std::vector calculate_column_widths(const dom::array& arr, - const std::vector& columns) const; - - /** Measure the actual formatted length of a value (for alignment) */ - size_t measure_value_length(const dom::element& elem) const; + /** Whether to pad this container's own brackets. */ + bool bracket_padding_for(const element_metrics& metrics) const; }; } // namespace internal @@ -11260,6 +12512,8 @@ class fractured_string_builder { #include #include #include +#include +#include namespace simdjson { namespace internal { @@ -11290,7 +12544,7 @@ inline element_metrics structure_analyzer::analyze_object(const dom::object& obj return analyze_object(obj, 0); } -inline element_metrics structure_analyzer::analyze_element(const dom::element& elem, size_t depth) { +inline element_metrics structure_analyzer::analyze_element(const dom::element& elem, size_t depth) const { switch (elem.type()) { case dom::element_type::ARRAY: { dom::array arr; @@ -11313,12 +12567,11 @@ inline element_metrics structure_analyzer::analyze_element(const dom::element& e return element_metrics{}; } -inline element_metrics structure_analyzer::analyze_scalar(const dom::element& elem) { +inline element_metrics structure_analyzer::analyze_scalar(const dom::element& elem) const { element_metrics metrics; metrics.complexity = 0; metrics.child_count = 0; metrics.can_inline = true; - metrics.recommended_layout = layout_mode::INLINE; switch (elem.type()) { case dom::element_type::STRING: { @@ -11367,7 +12620,7 @@ inline element_metrics structure_analyzer::analyze_scalar(const dom::element& el } inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, - size_t depth) { + size_t depth) const { element_metrics metrics; metrics.complexity = 1; // At least 1 for being an array metrics.estimated_inline_len = 2; // "[]" @@ -11392,35 +12645,31 @@ inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, // Complexity is 1 + max child complexity metrics.complexity = 1 + max_child_complexity; - // Check if can inline - metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && - (metrics.estimated_inline_len <= current_opts_->max_inline_length); - - // Check for uniform array (table formatting) - if (current_opts_->enable_table_format && - metrics.child_count >= current_opts_->min_table_rows) { - metrics.is_uniform_array = check_array_uniformity(arr, metrics.common_keys); + // Bracket padding "[ 1, 2 ]" vs "[1, 2]" + bool use_bracket_padding = (max_child_complexity >= 1) + ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding; + if (use_bracket_padding && metrics.child_count > 0) { + metrics.estimated_inline_len += 2; } - // Decide layout - if (metrics.child_count == 0) { - metrics.recommended_layout = layout_mode::INLINE; - } else if (metrics.can_inline) { - metrics.recommended_layout = layout_mode::INLINE; - } else if (metrics.is_uniform_array && !metrics.common_keys.empty()) { - metrics.recommended_layout = layout_mode::TABLE; - } else if (current_opts_->enable_compact_multiline && - max_child_complexity <= current_opts_->max_compact_array_complexity) { - metrics.recommended_layout = layout_mode::COMPACT_MULTILINE; - } else { - metrics.recommended_layout = layout_mode::EXPANDED; + // Check if can inline + metrics.can_inline = metrics.complexity <= current_opts_->max_inline_complexity; + + // Check for uniform array (table formatting, or aligned compact multiline). + bool wants_table_columns = + (current_opts_->enable_table_format && + max_child_complexity <= current_opts_->max_table_row_complexity) || + (current_opts_->enable_compact_multiline && + max_child_complexity <= current_opts_->max_compact_array_complexity); + if (wants_table_columns) { + metrics.is_uniform_array = check_array_uniformity(arr, metrics, depth); } return metrics; } inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, - size_t depth) { + size_t depth) const { element_metrics metrics; metrics.complexity = 1; metrics.estimated_inline_len = 2; // "{}" @@ -11447,16 +12696,15 @@ inline element_metrics structure_analyzer::analyze_object(const dom::object& obj metrics.complexity = 1 + max_child_complexity; - metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && - (metrics.estimated_inline_len <= current_opts_->max_inline_length); - - // Objects use inline or expanded (no table/compact for objects) - if (metrics.child_count == 0 || metrics.can_inline) { - metrics.recommended_layout = layout_mode::INLINE; - } else { - metrics.recommended_layout = layout_mode::EXPANDED; + // Bracket padding '{ "a": 1 }' vs '{"a": 1}' + bool use_bracket_padding = (max_child_complexity >= 1) + ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding; + if (use_bracket_padding && metrics.child_count > 0) { + metrics.estimated_inline_len += 2; } + metrics.can_inline = metrics.complexity <= current_opts_->max_inline_complexity; + return metrics; } @@ -11473,8 +12721,18 @@ inline size_t structure_analyzer::estimate_string_length(std::string_view s) con } inline size_t structure_analyzer::estimate_number_length(double d) const { - if (std::isnan(d) || std::isinf(d)) { + if (!std::isfinite(d)) { +#if SIMDJSON_ENABLE_NAN_INF + if (std::isnan(d)) { + return 3; // "NaN" + } else if (d < 0) { + return 9; // "-Infinity" + } else { + return 8; // "Infinity" + } +#else return 4; // "null" for invalid numbers +#endif } // Rough estimate: up to 17 significant digits + sign + decimal point + exponent char buf[32]; @@ -11505,115 +12763,290 @@ inline size_t structure_analyzer::estimate_number_length(uint64_t u) const { return len; } -inline bool structure_analyzer::check_array_uniformity(const dom::array& arr, - std::vector& common_keys) const { - common_keys.clear(); - - std::set shared_keys; - dom::object first_obj; - bool have_first = false; - size_t object_count = 0; +inline table_column_type structure_analyzer::classify_table_value(dom::element_type type) { + switch (type) { + case dom::element_type::OBJECT: return table_column_type::object; + case dom::element_type::ARRAY: return table_column_type::array; + case dom::element_type::INT64: + case dom::element_type::UINT64: + case dom::element_type::DOUBLE: return table_column_type::number; + case dom::element_type::NULL_VALUE: return table_column_type::unknown; + default: return table_column_type::simple; // string, bool + } +} - for (dom::element elem : arr) { - if (elem.type() != dom::element_type::OBJECT) { - return false; // Not all elements are objects +inline void structure_analyzer::classify_and_measure( + const std::vector>& values, + table_column_type& common, size_t& max_width) { + common = table_column_type::unknown; + max_width = 0; + for (const auto& v : values) { + table_column_type t = classify_table_value(v.first.type()); + if (t != table_column_type::unknown) { + if (common == table_column_type::unknown) common = t; + else if (t != common) common = table_column_type::mixed; } - - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - return false; + if (v.second) { + max_width = (std::max)(max_width, v.second->estimated_inline_len); } + } +} - std::set current_keys; - for (dom::key_value_pair field : obj) { - current_keys.insert(std::string(field.key)); +inline void structure_analyzer::build_table_columns( + const std::vector>& values, + std::vector& out_columns) const { + out_columns.clear(); + if (values.empty()) { + return; + } + + table_column_type common = table_column_type::unknown; + for (const auto& v : values) { + table_column_type t = classify_table_value(v.first.type()); + if (t == table_column_type::unknown) continue; + if (common == table_column_type::unknown) common = t; + else if (t != common) { common = table_column_type::mixed; break; } + } + if (common != table_column_type::object && common != table_column_type::array) { + return; + } + + std::vector>> per_column_values; + + if (common == table_column_type::object) { + std::unordered_map column_index; + // Last row (1-based) that filled each column, to detect duplicate keys. + std::vector column_last_row; + size_t row = 0; + for (const auto& v : values) { + if (v.first.type() != dom::element_type::OBJECT) continue; + dom::object obj; + if (v.first.get_object().get(obj) != SUCCESS) continue; + + row++; + size_t field_idx = 0; + for (dom::key_value_pair field : obj) { + auto it = column_index.find(field.key); + size_t col_idx; + if (it == column_index.end()) { + col_idx = out_columns.size(); + column_index.emplace(field.key, col_idx); + out_columns.emplace_back(); + out_columns.back().key.assign(field.key.data(), field.key.size()); + out_columns.back().has_key = true; + out_columns.back().key_width = estimate_string_length(field.key); + per_column_values.emplace_back(); + column_last_row.push_back(0); + } else { + col_idx = it->second; + } + // A row with a duplicate key cannot be laid out as a table: each + // column holds one value per row, so the later duplicates would be + // lost. Returning no columns also disables the aligned compact + // multiline layout for this array, which needs the same columns. + if (column_last_row[col_idx] == row) { + out_columns.clear(); + return; + } + column_last_row[col_idx] = row; + const element_metrics* field_metrics = (v.second && field_idx < v.second->children.size()) + ? &v.second->children[field_idx] : nullptr; + per_column_values[col_idx].emplace_back(field.value, field_metrics); + field_idx++; + } } + } else { // array: columns by position + for (const auto& v : values) { + if (v.first.type() != dom::element_type::ARRAY) continue; + dom::array sub_arr; + if (v.first.get_array().get(sub_arr) != SUCCESS) continue; - if (!have_first) { - shared_keys = current_keys; - first_obj = obj; - have_first = true; - } else { - // Check similarity threshold against the first object - double similarity = compute_object_similarity(first_obj, obj); - if (similarity < current_opts_->table_similarity_threshold) { - return false; // Objects are too dissimilar for table format + size_t idx = 0; + for (dom::element item : sub_arr) { + if (out_columns.size() <= idx) { + out_columns.emplace_back(); + per_column_values.emplace_back(); + } + const element_metrics* item_metrics = (v.second && idx < v.second->children.size()) + ? &v.second->children[idx] : nullptr; + per_column_values[idx].emplace_back(item, item_metrics); + idx++; } + } + } + + for (size_t i = 0; i < out_columns.size(); i++) { + table_column_type col_type; + size_t max_width; + classify_and_measure(per_column_values[i], col_type, max_width); + out_columns[i].type = col_type; + out_columns[i].plain_width = max_width; - // Intersect with current keys - std::set intersection; - std::set_intersection(shared_keys.begin(), shared_keys.end(), - current_keys.begin(), current_keys.end(), - std::inserter(intersection, intersection.begin())); - shared_keys = intersection; + if (col_type == table_column_type::object || col_type == table_column_type::array) { + build_table_columns(per_column_values[i], out_columns[i].children); } - object_count++; + out_columns[i].width = out_columns[i].children.empty() ? max_width : compute_columns_width(out_columns[i].children); } +} - if (object_count < current_opts_->min_table_rows) { - return false; +inline bool structure_analyzer::check_array_uniformity(const dom::array& arr, + element_metrics& metrics, + size_t depth) const { + std::vector> values; + values.reserve(metrics.child_count); + + size_t row_idx = 0; + for (dom::element elem : arr) { + const element_metrics* row_metrics = (row_idx < metrics.children.size()) ? &metrics.children[row_idx] : nullptr; + values.emplace_back(elem, row_metrics); + row_idx++; + } + + build_table_columns(values, metrics.table_columns); + if (metrics.table_columns.empty()) { + // Not uniformly object or array. Check for uniform scalar + table_column_type common; + size_t max_width; + classify_and_measure(values, common, max_width); + if (common != table_column_type::number && common != table_column_type::simple) { + return false; + } + metrics.scalar_column_type = common; + metrics.table_row_width = max_width; + metrics.table_row_width_full = max_width; + return true; } - // Require at least one common key for table formatting - if (shared_keys.empty()) { - return false; + metrics.table_row_width_full = compute_columns_width(metrics.table_columns); + + size_t row_indent_width = (depth + 1) * current_opts_->indent_spaces; + size_t budget = (row_indent_width + 1 >= current_opts_->max_total_line_length) + ? 0 : current_opts_->max_total_line_length - row_indent_width - 1; + + size_t width = metrics.table_row_width_full; + while (width > budget && flatten_deepest_columns(metrics.table_columns)) { + recompute_column_widths(metrics.table_columns); + width = compute_columns_width(metrics.table_columns); } - common_keys.assign(shared_keys.begin(), shared_keys.end()); + metrics.table_row_width = width; return true; } -inline double structure_analyzer::compute_object_similarity(const dom::object& a, - const dom::object& b) const { - std::set keys_a, keys_b; - for (dom::key_value_pair field : a) { - keys_a.insert(std::string(field.key)); +inline size_t structure_analyzer::compute_columns_width(const std::vector& columns) const { + size_t width = 2; // "{}" or "[]" + if (table_row_is_nested(columns) ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding) { + width += 2; } - for (dom::key_value_pair field : b) { - keys_b.insert(std::string(field.key)); + + for (const table_column& col : columns) { + if (col.has_key) { + width += col.key_width; + width += current_opts_->colon_padding ? 2 : 1; + } + width += col.width; + } + if (columns.size() > 1) { + width += (columns.size() - 1) * (current_opts_->comma_padding ? 2 : 1); + } + return width; +} + +inline size_t structure_analyzer::column_height(const table_column& column) { + size_t height = 0; + for (const table_column& child : column.children) { + height = (std::max)(height, column_height(child)); } + return column.children.empty() ? 0 : height + 1; +} - std::set intersection; - std::set_intersection(keys_a.begin(), keys_a.end(), - keys_b.begin(), keys_b.end(), - std::inserter(intersection, intersection.begin())); +inline bool structure_analyzer::flatten_deepest_columns(std::vector& columns) { + size_t max_height = 0; + for (const table_column& col : columns) { + max_height = (std::max)(max_height, column_height(col)); + } - std::set union_set; - std::set_union(keys_a.begin(), keys_a.end(), - keys_b.begin(), keys_b.end(), - std::inserter(union_set, union_set.begin())); + bool changed = false; + for (table_column& col : columns) { + if (column_height(col) != max_height || max_height == 0) continue; + if (max_height == 1) { + col.children.clear(); + col.width = col.plain_width; + changed = true; + } else { + changed |= flatten_deepest_columns(col.children); + } + } + return changed; +} - if (union_set.empty()) return 1.0; - return static_cast(intersection.size()) / static_cast(union_set.size()); +inline void structure_analyzer::recompute_column_widths(std::vector& columns) const { + for (table_column& col : columns) { + if (!col.children.empty()) { + recompute_column_widths(col.children); + col.width = compute_columns_width(col.children); + } + } } inline layout_mode structure_analyzer::decide_layout(const element_metrics& metrics, size_t depth, - size_t available_width) const { + const fractured_json_options& opts, + bool has_trailing_comma) { if (metrics.child_count == 0) { - return layout_mode::INLINE; + return layout_mode::single_line; } + long long signed_depth = static_cast(depth); + bool depth_allows_inline_or_compact = signed_depth > opts.always_expand_depth; + bool depth_allows_table = signed_depth >= opts.always_expand_depth; + // Check inline feasibility - size_t indent_width = depth * current_opts_->indent_spaces; - if (metrics.can_inline && - metrics.estimated_inline_len + indent_width <= available_width) { - return layout_mode::INLINE; + size_t reserved_width = depth * opts.indent_spaces + (has_trailing_comma ? 1 : 0); + if (depth_allows_inline_or_compact && metrics.can_inline && + metrics.estimated_inline_len + reserved_width <= opts.max_total_line_length) { + return layout_mode::single_line; } - // Check table mode - if (metrics.is_uniform_array && !metrics.common_keys.empty()) { - return layout_mode::TABLE; - } + // Rows (table's or compact multiline's) render one level deeper than the + // array itself. + size_t row_indent_width = (depth + 1) * opts.indent_spaces; // Check compact multiline - if (current_opts_->enable_compact_multiline && - metrics.complexity <= current_opts_->max_compact_array_complexity + 1) { - return layout_mode::COMPACT_MULTILINE; + // for uniform arrays fall back to table if we would have to flatten any formatting + bool compact_multiline_enabled = opts.enable_compact_multiline && + metrics.complexity <= opts.max_compact_array_complexity + 1 && + metrics.child_count >= opts.min_compact_array_row_items; + if (depth_allows_inline_or_compact && compact_multiline_enabled) { + bool aligned = metrics.is_uniform_array; + size_t comma_width = opts.comma_padding ? 2 : 1; + size_t avg_item_width; + if (aligned) { + avg_item_width = metrics.table_row_width_full + comma_width; + } else { + size_t sum = 0; + for (const element_metrics& child : metrics.children) { + sum += child.estimated_inline_len; + } + avg_item_width = comma_width + sum / metrics.child_count; + } + + size_t row_pack_space = (row_indent_width >= opts.max_total_line_length) + ? 0 : opts.max_total_line_length - row_indent_width; + if (avg_item_width * opts.min_compact_array_row_items <= row_pack_space) { + return layout_mode::compact_multiline; + } + } + + // Check Table mode + if (depth_allows_table && opts.enable_table_format && + metrics.is_uniform_array && + metrics.table_row_width + 1 + row_indent_width <= opts.max_total_line_length) { + return layout_mode::table; } - return layout_mode::EXPANDED; + return layout_mode::expanded; } // @@ -11621,10 +13054,10 @@ inline layout_mode structure_analyzer::decide_layout(const element_metrics& metr // inline fractured_formatter::fractured_formatter(const fractured_json_options& opts) - : options_(opts), column_widths_{} {} + : options_(opts) {} simdjson_inline void fractured_formatter::print_newline() { - if (current_layout_ == layout_mode::INLINE) { + if (current_layout_ == layout_mode::single_line) { return; // No newlines in inline mode } one_char('\n'); @@ -11632,7 +13065,7 @@ simdjson_inline void fractured_formatter::print_newline() { } simdjson_inline void fractured_formatter::print_indents(size_t depth) { - if (current_layout_ == layout_mode::INLINE) { + if (current_layout_ == layout_mode::single_line) { return; // No indentation in inline mode } for (size_t i = 0; i < depth * options_.indent_spaces; i++) { @@ -11654,26 +13087,10 @@ inline layout_mode fractured_formatter::get_layout_mode() const { return current_layout_; } -inline void fractured_formatter::set_depth(size_t depth) { - current_depth_ = depth; -} - -inline size_t fractured_formatter::get_depth() const { - return current_depth_; -} - inline void fractured_formatter::track_line_length(size_t chars) { current_line_length_ += chars; } -inline void fractured_formatter::reset_line_length() { - current_line_length_ = 0; -} - -inline size_t fractured_formatter::get_line_length() const { - return current_line_length_; -} - inline bool fractured_formatter::should_break_line(size_t upcoming_length) const { return (current_line_length_ + upcoming_length) > options_.max_total_line_length; } @@ -11682,39 +13099,6 @@ inline const fractured_json_options& fractured_formatter::options() const { return options_; } -inline void fractured_formatter::begin_table_row() { - in_table_mode_ = true; - current_column_ = 0; -} - -inline void fractured_formatter::end_table_row() { - in_table_mode_ = false; - current_column_ = 0; -} - -inline void fractured_formatter::set_column_widths(const std::vector& widths) { - column_widths_ = widths; -} - -inline size_t fractured_formatter::get_column_index() const { - return current_column_; -} - -inline void fractured_formatter::next_column() { - current_column_++; -} - -inline void fractured_formatter::align_to_column_width(size_t actual_width) { - if (current_column_ < column_widths_.size()) { - size_t target_width = column_widths_[current_column_]; - while (actual_width < target_width) { - one_char(' '); - actual_width++; - current_line_length_++; - } - } -} - // // Fractured String Builder Implementation // @@ -11753,19 +13137,20 @@ simdjson_inline std::string_view fractured_string_builder::str() const { inline void fractured_string_builder::format_element(const dom::element& elem, const element_metrics& metrics, - size_t depth) { + size_t depth, + bool has_trailing_comma) { switch (elem.type()) { case dom::element_type::ARRAY: { dom::array arr; if (elem.get_array().get(arr) == SUCCESS) { - format_array(arr, metrics, depth); + format_array(arr, metrics, depth, has_trailing_comma); } break; } case dom::element_type::OBJECT: { dom::object obj; if (elem.get_object().get(obj) == SUCCESS) { - format_object(obj, metrics, depth); + format_object(obj, metrics, depth, has_trailing_comma); } break; } @@ -11777,18 +13162,20 @@ inline void fractured_string_builder::format_element(const dom::element& elem, inline void fractured_string_builder::format_array(const dom::array& arr, const element_metrics& metrics, - size_t depth) { - switch (metrics.recommended_layout) { - case layout_mode::INLINE: + size_t depth, + bool has_trailing_comma) { + layout_mode layout = structure_analyzer::decide_layout(metrics, depth, options_, has_trailing_comma); + switch (layout) { + case layout_mode::single_line: format_array_inline(arr, metrics); break; - case layout_mode::COMPACT_MULTILINE: + case layout_mode::compact_multiline: format_array_compact_multiline(arr, metrics, depth); break; - case layout_mode::TABLE: + case layout_mode::table: format_array_as_table(arr, metrics, depth); break; - case layout_mode::EXPANDED: + case layout_mode::expanded: default: format_array_expanded(arr, metrics, depth); break; @@ -11797,8 +13184,7 @@ inline void fractured_string_builder::format_array(const dom::array& arr, inline void fractured_string_builder::format_array_inline(const dom::array& arr, const element_metrics& metrics) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); + scoped_single_line_mode single_line(format_); format_.start_array(); @@ -11812,60 +13198,69 @@ inline void fractured_string_builder::format_array_inline(const dom::array& arr, if (options_.comma_padding) { format_.print_space(); } - } else if (options_.simple_bracket_padding) { + } else if (bracket_padding_for(metrics)) { format_.print_space(); } first = false; - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); format_element(elem, child_metrics, 0); child_idx++; } - if (options_.simple_bracket_padding && !empty) { + if (bracket_padding_for(metrics) && !empty) { format_.print_space(); } format_.end_array(); - - format_.set_layout_mode(prev_layout); } inline void fractured_string_builder::format_array_compact_multiline(const dom::array& arr, const element_metrics& metrics, size_t depth) { + if (metrics.is_uniform_array) { + format_array_compact_multiline_aligned(arr, metrics, depth); + return; + } + format_.start_array(); format_.print_newline(); format_.print_indents(depth + 1); - size_t items_on_line = 0; bool first = true; + bool prev_item_was_expanded = false; size_t child_idx = 0; for (dom::element elem : arr) { + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); + if (!first) { format_.comma(); + format_.track_line_length(1); // Check if we should break to new line - if (items_on_line >= options_.max_items_per_line || - format_.should_break_line(20)) { // 20 is rough estimate for next item + if (prev_item_was_expanded || + format_.should_break_line(child_metrics.estimated_inline_len)) { format_.print_newline(); format_.print_indents(depth + 1); - items_on_line = 0; } else if (options_.comma_padding) { format_.print_space(); } } first = false; - // Format element inline - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(elem, child_metrics, depth + 1); - format_.set_layout_mode(prev_layout); + bool is_last = (child_idx + 1 == metrics.child_count); + layout_mode item_layout = structure_analyzer::decide_layout(child_metrics, depth + 1, options_, !is_last); + bool item_fits = item_layout == layout_mode::single_line; + if (item_fits) { + { + scoped_single_line_mode single_line(format_); + format_element(elem, child_metrics, depth + 1, !is_last); + } + format_.track_line_length(child_metrics.estimated_inline_len); + } else { + format_element(elem, child_metrics, depth + 1, !is_last); + } + prev_item_was_expanded = !item_fits; - items_on_line++; child_idx++; } @@ -11874,114 +13269,309 @@ inline void fractured_string_builder::format_array_compact_multiline(const dom:: format_.end_array(); } -inline void fractured_string_builder::format_array_as_table(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - const std::vector& columns = metrics.common_keys; - if (columns.empty()) { - format_array_expanded(arr, metrics, depth); - return; - } - - // Calculate column widths for alignment - std::vector col_widths = calculate_column_widths(arr, columns); - format_.set_column_widths(col_widths); +inline void fractured_string_builder::format_array_compact_multiline_aligned( + const dom::array& arr, const element_metrics& metrics, size_t depth) { + const std::vector& columns = metrics.table_columns; format_.start_array(); format_.print_newline(); + format_.print_indents(depth + 1); - bool first_row = true; + size_t indent_width = (depth + 1) * options_.indent_spaces; + size_t available_line_space = (indent_width >= options_.max_total_line_length) + ? 0 : options_.max_total_line_length - indent_width; + size_t comma_width = options_.comma_padding ? 2 : 1; + size_t remaining_line_space = available_line_space; + + bool first = true; size_t child_idx = 0; + for (dom::element elem : arr) { - if (!first_row) { - format_.comma(); - format_.print_newline(); - } - first_row = false; + bool needs_comma = (child_idx + 1 < metrics.child_count); + size_t space_needed = metrics.table_row_width_full + (needs_comma ? comma_width : 0); - format_.print_indents(depth + 1); - format_.begin_table_row(); + if (!first) { + if (remaining_line_space < space_needed) { + format_.print_newline(); + format_.print_indents(depth + 1); + remaining_line_space = available_line_space; + } else if (options_.comma_padding) { + format_.print_space(); + } + } + first = false; - // Format object as inline with aligned columns - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - child_idx++; - continue; + const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); + if (columns.empty()) { + format_table_scalar_row(elem, row_metrics, metrics.table_row_width_full, depth + 1, + metrics.scalar_column_type); + } else { + format_table_row(elem, row_metrics, columns, depth + 1); } + if (needs_comma) { + format_.comma(); + } + remaining_line_space -= (std::min)(remaining_line_space, space_needed); + child_idx++; + } - // Get child metrics for this row (object) - const element_metrics& row_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; + format_.print_newline(); + format_.print_indents(depth); + format_.end_array(); +} - format_.start_object(); - if (options_.simple_bracket_padding) { - format_.print_space(); - } +inline void fractured_string_builder::format_table_row_columns( + const std::vector& columns, + const std::vector& found, + const std::vector& values, + const std::vector& value_metrics, + size_t depth) { + const size_t num_columns = columns.size(); + size_t last_present_idx = num_columns; + for (size_t i = 0; i < num_columns; i++) { + if (found[i]) last_present_idx = i; + } - bool first_col = true; - const size_t num_columns = columns.size(); + size_t comma_width = options_.comma_padding ? 2 : 1; - for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { - const std::string& key = columns[col_idx]; - const bool is_last_col = (col_idx == num_columns - 1); + for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { + const table_column& column = columns[col_idx]; + const bool is_last_col = (col_idx == num_columns - 1); - if (!first_col) { - format_.comma(); - if (options_.comma_padding) { + if (found[col_idx]) { + if (column.has_key) { + format_.key(column.key); + if (options_.colon_padding) { format_.print_space(); } } - first_col = false; - // Write key - format_.key(key); - if (options_.colon_padding) { - format_.print_space(); - } + bool needs_comma = !is_last_col && (col_idx < last_present_idx); - // Find the value for this key and its metrics - dom::element value; - bool found = false; - size_t field_idx = 0; - for (dom::key_value_pair field : obj) { - if (field.key == key) { - value = field.value; - found = true; - break; + if (!column.children.empty()) { + // Recurses into this cell's own columns instead of a plain value; + // every row aligns those the same way (blank-padding missing + // ones), so the result is always exactly column.width wide + // no padding needed afterward, unlike the leaf case below. + if (column.type == table_column_type::object) { + dom::object sub_obj; + if (values[col_idx].get_object().get(sub_obj) == SUCCESS) { + const element_metrics& sub_metrics = child_metrics_at(value_metrics[col_idx]); + format_table_object_row(sub_obj, sub_metrics, column.children, depth); + } + } else { + dom::array sub_arr; + if (values[col_idx].get_array().get(sub_arr) == SUCCESS) { + const element_metrics& sub_metrics = child_metrics_at(value_metrics[col_idx]); + format_table_array_row(sub_arr, sub_metrics, column.children, depth); + } } - field_idx++; + // value is already padded + if (needs_comma) { + format_.comma(); + if (options_.comma_padding) { + format_.print_space(); + } + } + } else { + const element_metrics& vm = child_metrics_at(value_metrics[col_idx]); + format_table_leaf_value(values[col_idx], vm, column.width, column.type, needs_comma, + /*add_comma_space=*/true, depth); } - // Write value - if (found) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - const element_metrics& value_metrics = (field_idx < row_metrics.children.size()) - ? row_metrics.children[field_idx] : element_metrics{}; - format_element(value, value_metrics, depth + 1); - format_.set_layout_mode(prev_layout); - } else { - format_.null_atom(); + if (!is_last_col && !needs_comma) { + // Found, but no more real values follow: blank space where a comma would go. + for (size_t i = 0; i < comma_width; i++) { + format_.one_char(' '); + } + } + } else { + size_t slot_width = column.width; + if (column.has_key) { + slot_width += column.key_width + (options_.colon_padding ? 2 : 1); + } + for (size_t i = 0; i < slot_width; i++) { + format_.one_char(' '); } - // Only pad non-last columns to align values across rows if (!is_last_col) { - size_t actual_len = found ? measure_value_length(value) : 4; // 4 for "null" - size_t target_width = col_widths[col_idx]; - while (actual_len < target_width) { + for (size_t i = 0; i < comma_width; i++) { format_.one_char(' '); - actual_len++; } } + } + } +} - format_.next_column(); +inline void fractured_string_builder::format_table_object_row( + const dom::object& obj, const element_metrics& row_metrics, + const std::vector& columns, size_t depth) { + const size_t num_columns = columns.size(); + std::vector found(num_columns, false); + std::vector values(num_columns); + std::vector value_metrics(num_columns, nullptr); + + for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { + size_t field_idx = 0; + for (dom::key_value_pair field : obj) { + if (field.key == columns[col_idx].key) { + found[col_idx] = true; + values[col_idx] = field.value; + value_metrics[col_idx] = (field_idx < row_metrics.children.size()) + ? &row_metrics.children[field_idx] : nullptr; + break; + } + field_idx++; } + } - if (options_.simple_bracket_padding) { - format_.print_space(); + bool nested = table_row_is_nested(columns); + format_.start_object(); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_table_row_columns(columns, found, values, value_metrics, depth); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_.end_object(); +} + +inline void fractured_string_builder::format_table_array_row( + const dom::array& arr, const element_metrics& row_metrics, + const std::vector& columns, size_t depth) { + const size_t num_columns = columns.size(); + std::vector found(num_columns, false); + std::vector values(num_columns); + std::vector value_metrics(num_columns, nullptr); + + size_t idx = 0; + for (dom::element item : arr) { + if (idx >= num_columns) break; + found[idx] = true; + values[idx] = item; + value_metrics[idx] = (idx < row_metrics.children.size()) ? &row_metrics.children[idx] : nullptr; + idx++; + } + + bool nested = table_row_is_nested(columns); + format_.start_array(); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_table_row_columns(columns, found, values, value_metrics, depth); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_.end_array(); +} + +inline void fractured_string_builder::format_table_row( + const dom::element& elem, const element_metrics& row_metrics, + const std::vector& columns, size_t depth) { + if (elem.type() == dom::element_type::ARRAY) { + dom::array arr; + if (elem.get_array().get(arr) == SUCCESS) { + format_table_array_row(arr, row_metrics, columns, depth); + } + } else { + dom::object obj; + if (elem.get_object().get(obj) == SUCCESS) { + format_table_object_row(obj, row_metrics, columns, depth); + } + } +} + +inline bool fractured_string_builder::comma_goes_before_padding(table_column_type column_type) const { + switch (options_.comma_placement) { + case table_comma_placement::before_padding: return true; + case table_comma_placement::after_padding: return false; + case table_comma_placement::before_padding_except_numbers: + default: + return column_type != table_column_type::number; + } +} + +inline void fractured_string_builder::format_table_leaf_value( + const dom::element& elem, const element_metrics& vm, size_t width, + table_column_type column_type, bool needs_comma, bool add_comma_space, size_t depth) { + bool comma_before_pad = needs_comma && comma_goes_before_padding(column_type); + bool comma_after_pad = needs_comma && !comma_before_pad; + + bool right_align = column_type == table_column_type::number && + options_.number_alignment == number_list_alignment::right; + + size_t value_len = vm.estimated_inline_len; + size_t left_pad = 0; + size_t right_pad = 0; + if (right_align) { + left_pad = (width > value_len) ? width - value_len : 0; + comma_before_pad = needs_comma; + comma_after_pad = false; + } else { + right_pad = (width > value_len) ? width - value_len : 0; + } + + for (size_t i = 0; i < left_pad; i++) { + format_.one_char(' '); + } + + { + scoped_single_line_mode single_line(format_); + format_element(elem, vm, depth); + } + + if (comma_before_pad) { + format_.comma(); + } + for (size_t i = 0; i < right_pad; i++) { + format_.one_char(' '); + } + if (comma_after_pad) { + format_.comma(); + } + if (needs_comma && add_comma_space && options_.comma_padding) { + format_.print_space(); + } +} + +inline void fractured_string_builder::format_table_scalar_row( + const dom::element& elem, const element_metrics& row_metrics, size_t width, size_t depth, + table_column_type column_type) { + format_table_leaf_value(elem, row_metrics, width, column_type, + /*needs_comma=*/false, /*add_comma_space=*/false, depth); +} + +inline void fractured_string_builder::format_array_as_table(const dom::array& arr, + const element_metrics& metrics, + size_t depth) { + if (!metrics.is_uniform_array) { + format_array_expanded(arr, metrics, depth); + return; + } + const std::vector& columns = metrics.table_columns; + + format_.start_array(); + format_.print_newline(); + + bool first_row = true; + size_t child_idx = 0; + for (dom::element elem : arr) { + if (!first_row) { + format_.comma(); + format_.print_newline(); + } + first_row = false; + + format_.print_indents(depth + 1); + + const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); + if (columns.empty()) { + format_table_scalar_row(elem, row_metrics, metrics.table_row_width, depth + 1, + metrics.scalar_column_type); + } else { + format_table_row(elem, row_metrics, columns, depth + 1); } - format_.end_object(); - format_.end_table_row(); child_idx++; } @@ -12008,9 +13598,9 @@ inline void fractured_string_builder::format_array_expanded(const dom::array& ar format_.print_newline(); format_.print_indents(depth + 1); - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(elem, child_metrics, depth + 1); + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); + bool is_last = (child_idx + 1 == metrics.child_count); + format_element(elem, child_metrics, depth + 1, !is_last); child_idx++; } @@ -12023,8 +13613,10 @@ inline void fractured_string_builder::format_array_expanded(const dom::array& ar inline void fractured_string_builder::format_object(const dom::object& obj, const element_metrics& metrics, - size_t depth) { - if (metrics.recommended_layout == layout_mode::INLINE || metrics.can_inline) { + size_t depth, + bool has_trailing_comma) { + layout_mode layout = structure_analyzer::decide_layout(metrics, depth, options_, has_trailing_comma); + if (layout == layout_mode::single_line) { format_object_inline(obj, metrics); } else { format_object_expanded(obj, metrics, depth); @@ -12033,8 +13625,7 @@ inline void fractured_string_builder::format_object(const dom::object& obj, inline void fractured_string_builder::format_object_inline(const dom::object& obj, const element_metrics& metrics) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); + scoped_single_line_mode single_line(format_); format_.start_object(); @@ -12049,7 +13640,7 @@ inline void fractured_string_builder::format_object_inline(const dom::object& ob if (options_.comma_padding) { format_.print_space(); } - } else if (options_.simple_bracket_padding) { + } else if (bracket_padding_for(metrics)) { format_.print_space(); } first = false; @@ -12058,18 +13649,15 @@ inline void fractured_string_builder::format_object_inline(const dom::object& ob if (options_.colon_padding) { format_.print_space(); } - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); format_element(field.value, child_metrics, 0); child_idx++; } - if (options_.simple_bracket_padding && !empty) { + if (bracket_padding_for(metrics) && !empty) { format_.print_space(); } format_.end_object(); - - format_.set_layout_mode(prev_layout); } inline void fractured_string_builder::format_object_expanded(const dom::object& obj, @@ -12094,9 +13682,9 @@ inline void fractured_string_builder::format_object_expanded(const dom::object& if (options_.colon_padding) { format_.print_space(); } - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(field.value, child_metrics, depth + 1); + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); + bool is_last = (child_idx + 1 == metrics.child_count); + format_element(field.value, child_metrics, depth + 1, !is_last); child_idx++; } @@ -12152,97 +13740,8 @@ inline void fractured_string_builder::format_scalar(const dom::element& elem) { } } -inline size_t fractured_string_builder::measure_value_length(const dom::element& elem) const { - switch (elem.type()) { - case dom::element_type::STRING: { - std::string_view str; - if (elem.get_string().get(str) == SUCCESS) { - // Count actual escaped length - size_t len = 2; // quotes - for (char c : str) { - if (c == '"' || c == '\\' || static_cast(c) < 32) { - len += 2; // escape sequence - } else { - len += 1; - } - } - return len; - } - return 2; - } - case dom::element_type::INT64: { - int64_t val; - if (elem.get_int64().get(val) == SUCCESS) { - if (val == 0) return 1; - // Handle INT64_MIN specially to avoid overflow when negating - if (val == INT64_MIN) return 20; // "-9223372036854775808" is 20 characters - size_t len = (val < 0) ? 1 : 0; - int64_t abs_val = (val < 0) ? -val : val; - while (abs_val > 0) { len++; abs_val /= 10; } - return len; - } - return 1; - } - case dom::element_type::UINT64: { - uint64_t val; - if (elem.get_uint64().get(val) == SUCCESS) { - if (val == 0) return 1; - size_t len = 0; - while (val > 0) { len++; val /= 10; } - return len; - } - return 1; - } - case dom::element_type::DOUBLE: { - double val; - if (elem.get_double().get(val) == SUCCESS) { - char buf[32]; - int len = snprintf(buf, sizeof(buf), "%.17g", val); - return len > 0 ? static_cast(len) : 1; - } - return 1; - } - case dom::element_type::BOOL: { - bool val; - if (elem.get_bool().get(val) == SUCCESS) { - return val ? 4 : 5; // "true" or "false" - } - return 5; - } - case dom::element_type::NULL_VALUE: - return 4; // "null" - default: - return 4; - } -} - -inline std::vector fractured_string_builder::calculate_column_widths( - const dom::array& arr, - const std::vector& columns) const { - - std::vector widths(columns.size(), 0); - - for (dom::element elem : arr) { - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - continue; - } - - for (size_t col_idx = 0; col_idx < columns.size(); col_idx++) { - const std::string& key = columns[col_idx]; - - for (dom::key_value_pair field : obj) { - if (field.key == key) { - // Measure actual value length - size_t len = measure_value_length(field.value); - widths[col_idx] = (std::max)(widths[col_idx], len); - break; - } - } - } - } - - return widths; +inline bool fractured_string_builder::bracket_padding_for(const element_metrics& metrics) const { + return metrics.complexity >= 2 ? options_.nested_bracket_padding : options_.simple_bracket_padding; } } // namespace internal @@ -12282,19 +13781,6 @@ std::string fractured_json(simdjson_result x, const fractured_json_options& o } #endif -// Explicit template instantiations for common types -template std::string fractured_json(dom::element x); -template std::string fractured_json(dom::element x, const fractured_json_options& options); -template std::string fractured_json(dom::array x); -template std::string fractured_json(dom::array x, const fractured_json_options& options); -template std::string fractured_json(dom::object x); -template std::string fractured_json(dom::object x, const fractured_json_options& options); - -#if SIMDJSON_EXCEPTIONS -template std::string fractured_json(simdjson_result x); -template std::string fractured_json(simdjson_result x, const fractured_json_options& options); -#endif - // // String-based API for formatting any JSON string // @@ -12662,6 +14148,8 @@ enum instruction_set { LASX = 0x40000, //RVV = 0x80000, RVV_VLS = 0x100000, + SVE = 0x200000, + SVE2 = 0x400000, }; } // namespace internal @@ -13774,7 +15262,7 @@ class implementation final : public simdjson::implementation { simdjson_inline implementation() : simdjson::implementation( "rvv_vls", "RISC-V V extension", - 0 + internal::instruction_set::RVV_VLS ) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, @@ -13933,7 +15421,14 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { /* result might be undefined when input_num is zero */ simdjson_inline int count_ones(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#else + // if the system supports SVE or CSSC, __builtin_popcountll + // might be compiled to fewer single instructions. For CSSC, + // __builtin_popcountll is compiled to a single instruction. + return __builtin_popcountll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO } @@ -13970,15 +15465,6 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace arm64 @@ -14242,6 +15728,7 @@ namespace { return vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); } + // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -14890,6 +16377,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -14901,6 +16391,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -14937,6 +16449,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace arm64 @@ -15027,7 +16604,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -15206,6 +16783,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -15245,6 +16823,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -15501,6 +17091,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -15538,6 +17341,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -15556,6 +17369,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -15572,26 +17407,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -15680,7 +17562,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -15763,15 +17645,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -15802,7 +17686,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -15851,7 +17749,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -15950,7 +17848,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16048,7 +17946,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16103,7 +18001,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16189,7 +18087,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16229,11 +18127,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -16244,9 +18151,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -16295,67 +18201,12 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { // // Check for minus sign // @@ -16367,91 +18218,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + if ( p == src ) { - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -16462,9 +18242,239 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { return INCORRECT_TYPE; } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t *start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -16513,6 +18523,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -17109,6 +19212,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -17120,6 +19226,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -17156,6 +19284,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace fallback @@ -17246,7 +19439,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -17425,6 +19618,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -17464,6 +19658,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -17720,6 +19926,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -17757,6 +20176,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -17775,6 +20204,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -17791,26 +20242,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -17899,7 +20397,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -17982,15 +20480,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -18021,7 +20521,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -18070,7 +20584,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -18169,7 +20683,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18267,7 +20781,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18322,7 +20836,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18408,7 +20922,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18448,11 +20962,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -18463,9 +20986,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -18514,6 +21036,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -18666,11 +21279,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -18681,9 +21308,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -18732,6 +21358,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -19053,16 +21772,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace haswell @@ -19815,6 +22524,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -19826,6 +22538,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -19862,6 +22596,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace haswell @@ -19952,7 +22751,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -20131,6 +22930,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -20170,6 +22970,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -20426,6 +23238,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -20463,6 +23488,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -20481,6 +23516,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -20497,26 +23554,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -20605,7 +23709,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -20688,15 +23792,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -20727,7 +23833,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -20776,7 +23896,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -20875,7 +23995,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -20973,7 +24093,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -21028,7 +24148,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -21114,7 +24234,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -21154,11 +24274,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -21169,9 +24298,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -21220,6 +24348,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -21372,11 +24591,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -21387,9 +24620,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -21438,6 +24670,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -21756,16 +25081,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace icelake @@ -22521,6 +25836,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -22532,6 +25850,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -22568,6 +25908,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace icelake @@ -22658,7 +26063,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -22837,6 +26242,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -22876,6 +26282,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -23132,6 +26550,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -23169,6 +26800,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -23187,6 +26828,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -23203,26 +26866,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -23311,7 +27021,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -23394,15 +27104,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -23433,7 +27145,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -23482,7 +27208,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -23581,7 +27307,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23679,7 +27405,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23734,7 +27460,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23820,7 +27546,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23860,11 +27586,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -23875,9 +27610,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -23926,6 +27660,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -24078,11 +27903,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -24093,9 +27932,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -24144,6 +27982,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -24434,16 +28365,6 @@ simdjson_inline int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace ppc64 @@ -25342,6 +29263,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -25353,6 +29277,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -25389,6 +29335,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace ppc64 @@ -25479,7 +29490,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -25658,6 +29669,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -25697,6 +29709,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -25953,6 +29977,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -25990,6 +30227,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -26008,6 +30255,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -26024,26 +30293,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -26132,7 +30448,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -26215,15 +30531,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -26254,7 +30572,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -26303,7 +30635,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -26402,7 +30734,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26500,7 +30832,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26555,7 +30887,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26641,7 +30973,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26681,11 +31013,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -26696,9 +31037,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -26747,67 +31087,12 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { // // Check for minus sign // @@ -26819,91 +31104,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + if ( p == src ) { - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -26914,9 +31128,239 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { return INCORRECT_TYPE; } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t *start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -26965,6 +31409,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -27269,16 +31806,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -27857,16 +32384,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -28480,6 +32997,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -28491,6 +33011,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -28527,6 +33069,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace westmere @@ -28617,7 +33224,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -28796,6 +33403,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -28835,6 +33443,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -29091,6 +33711,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -29128,6 +33961,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -29146,6 +33989,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -29162,26 +34027,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -29270,7 +34182,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -29353,15 +34265,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -29392,7 +34306,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -29441,7 +34369,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -29540,7 +34468,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29638,7 +34566,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29693,7 +34621,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29779,7 +34707,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29819,11 +34747,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -29834,9 +34771,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -29885,6 +34821,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -30037,11 +35064,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -30052,9 +35093,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -30103,6 +35143,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -30368,10 +35501,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lasx @@ -31118,6 +36247,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -31129,6 +36261,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -31165,6 +36319,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lasx @@ -31255,7 +36474,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -31434,6 +36653,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -31473,6 +36693,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -31729,6 +36961,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -31766,6 +37211,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -31784,6 +37239,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -31800,26 +37277,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -31908,7 +37432,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -31991,15 +37515,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -32030,7 +37556,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -32079,7 +37619,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -32178,7 +37718,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32276,7 +37816,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32331,7 +37871,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32417,7 +37957,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32457,11 +37997,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -32472,9 +38021,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -32523,6 +38071,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -32675,11 +38314,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -32690,9 +38343,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -32741,6 +38393,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -33002,10 +38747,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lsx @@ -33734,6 +39475,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -33745,6 +39489,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -33781,6 +39547,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lsx @@ -33871,7 +39702,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -34050,6 +39881,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -34089,6 +39921,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -34345,6 +40189,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -34382,6 +40439,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -34400,6 +40467,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -34416,26 +40505,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -34524,7 +40660,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -34607,15 +40743,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -34646,7 +40784,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -34695,7 +40847,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -34794,7 +40946,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -34892,7 +41044,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -34947,7 +41099,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -35033,7 +41185,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -35073,11 +41225,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -35088,9 +41249,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -35139,6 +41299,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -35291,11 +41542,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -35306,9 +41571,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -35357,6 +41621,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -35622,11 +41979,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { return __builtin_popcountll(input_num); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace rvv_vls @@ -36367,6 +42719,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -36378,6 +42733,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -36414,6 +42791,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace rvv_vls @@ -36504,7 +42946,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -36683,6 +43125,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -36722,6 +43165,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -36978,6 +43433,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -37015,6 +43683,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -37033,6 +43711,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -37049,26 +43749,73 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. + // Prior related work: jsonifier parses integers as eight-digit SWAR words + // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This + // takes one such word with parse_eight_digits_unrolled, the routine + // simdjson uses for long fractions. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -37157,7 +43904,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -37240,15 +43987,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -37279,7 +44028,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -37328,7 +44091,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -37427,7 +44190,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37525,7 +44288,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37580,7 +44343,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37666,7 +44429,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37706,11 +44469,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -37721,9 +44493,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37772,67 +44543,12 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { // // Check for minus sign // @@ -37844,91 +44560,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + if ( p == src ) { - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -37939,9 +44584,239 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { return INCORRECT_TYPE; } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t *start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37990,6 +44865,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -38172,6 +45140,376 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ // Otherwise, amalgamation will fail. /* skipped duplicate #include "simdjson/concepts.h" */ /* skipped duplicate #include "simdjson/dom/fractured_json.h" */ +/* including simdjson/annotations.h: #include "simdjson/annotations.h" */ +/* begin file simdjson/annotations.h */ +#ifndef SIMDJSON_ANNOTATIONS_H +#define SIMDJSON_ANNOTATIONS_H + +/** + * @file annotations.h + * @brief Provides compile-time annotations for simdjson structures. + * This header defines annotations that can be applied to data members of structures + * (and to the structures and enumerations themselves) to control how they are + * serialized/deserialized with simdjson. The set of annotations is modelled after + * the attributes of the Rust serde library. + * + * Member annotations: + * + * [[= simdjson::rename<"name">]] use "name" as the JSON key + * [[= simdjson::alias<"a", "b">]] also accept "a" and "b" when deserializing + * [[= simdjson::skip]] never serialize nor deserialize + * [[= simdjson::skip_serializing]] never serialize + * [[= simdjson::skip_deserializing]] never deserialize (keeps its current value) + * [[= simdjson::skip_serializing_if]] do not serialize when pred(value) is true + * [[= simdjson::default_value]] a missing key is not an error + * [[= simdjson::default_from]] a missing key sets the member to factory() + * [[= simdjson::with]] custom (de)serialization via Adapter + * [[= simdjson::flatten]] inline the members of a nested structure + * + * Structure (container) annotations: + * + * [[= simdjson::rename_all]] rename every member + * [[= simdjson::default_value]] no missing key is an error + * [[= simdjson::deny_unknown_fields]] unknown keys are a deserialization error + * [[= simdjson::transparent]] (de)serialize as the single member + * + * Enumeration annotations: rename_all on the enumeration, rename and alias on the + * enumerators (e.g., `enum class color { red [[= simdjson::rename<"RED">]] };`). + * + * This is currently experimental and subject to change (syntax and semantics may evolve). + */ + +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include + +namespace simdjson { + +// Structural compile-time string -- char array avoids the pointer-based +// 'reflect_constant failed' that occurs with const char* / string_view members. +template +struct fixed_string { + char data[N]; + + consteval fixed_string(const char (&s)[N]) noexcept { + for (size_t i = 0; i < N; ++i) { data[i] = s[i]; } + } + + consteval std::string_view view() const noexcept { return {data, N - 1}; } + + consteval bool operator==(const fixed_string&) const noexcept = default; +}; + +/** + * Naming conventions for simdjson::rename_all, mirroring serde's rename_all. + * Except for lowercase and uppercase, the C++ identifier is first split into + * words at underscores and at case changes ("userId", "user_id" and "UserId" all + * give the words "user" and "id"; "HTTPServer" gives "HTTP" and "Server"), and + * the words are then joined according to the convention. + */ +enum class case_style { + lowercase, ///< every letter lowercased, nothing else changes: userId -> userid + uppercase, ///< every letter uppercased, nothing else changes: user_id -> USER_ID + pascal_case, ///< UserId + camel_case, ///< userId + snake_case, ///< user_id + screaming_snake_case, ///< USER_ID + kebab_case, ///< user-id + screaming_kebab_case ///< USER-ID +}; + +namespace detail { + template + struct rename_t { + static constexpr auto name = Name; + // Exposed as a pointer and a size: std::meta::extract requires structural types. + static constexpr const char *key_data = Name.data; + static constexpr size_t key_size = Name.view().size(); + }; + template + struct alias_t { + static_assert(sizeof...(Names) > 0, "simdjson::alias requires at least one name"); + static constexpr std::string_view keys[] = {Names.view()...}; + static constexpr const std::string_view *keys_data = keys; + static constexpr size_t keys_count = sizeof...(Names); + }; + struct skip_tag {}; + struct skip_serializing_tag {}; + struct skip_deserializing_tag {}; + template + struct skip_serializing_if_t { + static constexpr auto predicate = Predicate; + }; + struct default_value_tag {}; + template + struct default_from_t { + static constexpr auto factory = Factory; + }; + template + struct with_t { + using adapter = Adapter; + }; + template + struct rename_all_t { + static constexpr case_style style = Style; + }; + struct deny_unknown_fields_tag {}; + struct transparent_tag {}; + struct flatten_tag {}; + + // Predicates usable with skip_serializing_if. + struct is_none_t { + template + constexpr bool operator()(const T& v) const noexcept { return !v; } + }; + struct is_empty_t { + template + constexpr bool operator()(const T& v) const noexcept { return v.empty(); } + }; +} // namespace detail + +// Usage: [[= simdjson::rename<"first_name">]] std::string firstName; +template +inline constexpr detail::rename_t rename{}; + +// Usage: [[= simdjson::alias<"userName", "login">]] std::string user_name; +// The aliases are accepted (in addition to the regular key) when deserializing. +// Serialization always uses the regular key. If the JSON object contains more +// than one of the names, which one is used is unspecified. +template +inline constexpr detail::alias_t alias{}; + +// Usage: [[= simdjson::skip]] int internalCache; +inline constexpr detail::skip_tag skip{}; + +// Usage: [[= simdjson::skip_serializing]] std::string password; +inline constexpr detail::skip_serializing_tag skip_serializing{}; + +// Usage: [[= simdjson::skip_deserializing]] int computed; +// The member is never assigned during deserialization (it keeps its current +// value) and a matching key in the JSON input is treated as unknown. +inline constexpr detail::skip_deserializing_tag skip_deserializing{}; + +// Usage: [[= simdjson::skip_serializing_if]] std::optional x; +// The predicate is called with the member value; when it returns true, the key +// is omitted from the output. +template +inline constexpr detail::skip_serializing_if_t skip_serializing_if{}; + +// Predicate: true for an empty std::optional, a null smart pointer, etc. +inline constexpr detail::is_none_t is_none{}; +// Predicate: true for an empty string or container. +inline constexpr detail::is_empty_t is_empty{}; + +// Usage: [[= simdjson::default_value]] int port = 8080; +// When the key is missing from the JSON input, the member is left untouched +// (with get(), it keeps its default member initializer) instead of reporting +// NO_SUCH_FIELD. Applied to a structure, it applies to all of its members. +inline constexpr detail::default_value_tag default_value{}; + +// Usage: [[= simdjson::default_from]] int port; +// When the key is missing from the JSON input, the member is assigned the +// result of calling the factory (a constexpr callable taking no argument, such +// as a captureless lambda or a pointer to a function). +template +inline constexpr detail::default_from_t default_from{}; + +// Usage: [[= simdjson::with]] std::chrono::system_clock::time_point t; +// Adapter is a type that provides one or both of +// static void serialize(simdjson::builder::string_builder &b, const T &value); +// static simdjson::error_code deserialize(simdjson::ondemand::value &v, T &out); +// (the parameters may also be declared auto&). When one of them is missing, the +// default behaviour is used in that direction. +template +inline constexpr detail::with_t with{}; + +// Usage: [[= simdjson::flatten]] pagination page; +// The members of the nested structure are (de)serialized as if they were members +// of the enclosing structure: {"id":1,"limit":10,"offset":0} rather than +// {"id":1,"page":{"limit":10,"offset":0}}. The nested structure's own annotations +// (rename_all, default_value, ...) apply to its members. +inline constexpr detail::flatten_tag flatten{}; + +// Usage: struct [[= simdjson::rename_all]] S {...}; +// Also applies to enumerations. An explicit rename on a member takes precedence. +template +inline constexpr detail::rename_all_t