From dca05ad53e898a66744afb350c566819e7f952cc Mon Sep 17 00:00:00 2001 From: Seth Parker Date: Sat, 19 Sep 2026 07:45:31 -0400 Subject: [PATCH 1/3] merge-items: Merge output directories, not just items.json `preppy-merge-items` now auto-detects its input shape. Directories (or their index.json) take a new directory-merge path: each per-object subdirectory is copied into the output directory and the index.json objects[] lists are combined. Plain .json files still take the legacy items.json document-list path; mixing the two is a CLI error. Object id is the merge key. Duplicate ids across sources are an error unless --overwrite (last source wins, replacing the whole object directory so stale hashed generations don't linger); two different objects claiming the same subdirectory is always an error. The merge is planned up front, so a collision fails before anything is copied. Subdirectory names are preserved, keeping each entry's relative manifest/thumb/variant URIs valid. The tool keeps its `merge_items` name -- the implementation plan's note about renaming it to `merge_index` is retired. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 2 + README.md | 20 ++++ docs/implementation-plan.md | 4 +- preppy/apps/merge_items.py | 193 ++++++++++++++++++++++++++++++++---- tests/test_merge_items.py | 134 +++++++++++++++++++++++++ 5 files changed, 331 insertions(+), 22 deletions(-) create mode 100644 tests/test_merge_items.py diff --git a/CLAUDE.md b/CLAUDE.md index 6acb169..707c3a0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,6 +31,7 @@ npm install --prefix preppy/node # KTX2 embed helper deps (once) preppy -i config.json -o out/ # batch: variants → self-contained glbs + manifest.json + index.json preppy-check-tools # report external toolchain status preppy-obj2glb -i mesh.obj -o mesh.glb # LEGACY single OBJ → Draco GLB (deprecated) +preppy-merge-items out1/ out2/ -o merged/ # merge output dirs (assets + index.json) preppy-merge-items a.json b.json -o merged.json # LEGACY items.json merge (deprecated) ``` @@ -48,6 +49,7 @@ Console entrypoints in `preppy/apps/` are thin argparse CLIs over the library mo - **`cache.py`** — content hash over **inputs + config + tool versions** (never the output glb — basis encoding is non-deterministic), `hashed_name()`, and `prune()`. - **`manifest.py`** (replaced `voyager.py`) — pure builders for the per-object `manifest.json` (flat `variants[] {id,label,uri,default}` + object metadata, `units:"cm"`, per-variant overrides) and the optional `index.json`. - **`apps/file_prep.py`** — the orchestrator. Per object, per **variant** (no grouping): resolve texture(s) → normalize + encode each → gltfpack → optional Hausdorff gate → embed all by name → one self-contained glb → manifest entry. Emits `manifest.json` per object + a top-level `index.json`. The default-variant thumbnail is a rendered model preview (`preview.render_preview`, `_render_thumbnail` helper) that falls back to a texture crop (`--thumbnail-mode texture`, or automatically when the render toolchain is unavailable). +- **`apps/merge_items.py`** — merges separately-built catalogs. Input shape is auto-detected: output directories (or their `index.json`) are merged by copying each per-object subdirectory into `-o` and combining the `index.json` `objects[]` (merge key is object `id`; collisions are planned up front so nothing is copied before the whole merge validates). Plain `.json` inputs take the deprecated `items.json` document-list path. - **`convert.py` + `apps/obj_to_glb.py`** — the deprecated legacy OBJ→Draco-GLB path (kept until removed). ### Input config format diff --git a/README.md b/README.md index 7558b0b..f8d0f13 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,26 @@ texture`) automatically. Run `preppy -h` for the complete list. +### Merging catalogs + +Objects built in separate runs (or on separate machines) combine into one +servable catalog with `preppy-merge-items`. Each per-object subdirectory is +copied into the output directory and the `index.json` listings are merged: + +```shell +preppy-merge-items run1/ run2/ run3/ -o catalog/ +``` + +Object `id` is the merge key, and relative `manifest`/`thumb` URIs are preserved +by keeping each object's subdirectory name. Duplicate ids across sources are an +error — pass `--overwrite` to let the last source win, which is how a rebuilt +object replaces its older assets in place. The whole merge is validated before +anything is copied, and `--no-sort` keeps source order instead of sorting by +title. + +Passing `.json` files instead of directories runs the deprecated `items.json` +document-list merge (`--merge-duplicates`). + ### Hosting / caching Serve the catalog with a two-tier cache policy so cache-busting is safe: diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 8478895..1130e55 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -160,7 +160,9 @@ mogrify — cheap, no offscreen GL. (Loose end: rendered 3D thumbs later.) ### A6. CLI Keep console-script entry; add `--ktx2-mode {uastc,etc1s}`, `--decimate-error`, -`--no-decimate`, keep `--uri`, `--keep-tmp`. `merge_items` → `merge_index` later. +`--no-decimate`, keep `--uri`, `--keep-tmp`. `merge_items` keeps its name (no +rename to `merge_index`); it gained the output-directory merge alongside the +legacy `items.json` one. --- diff --git a/preppy/apps/merge_items.py b/preppy/apps/merge_items.py index 5fbcb9f..92152e8 100644 --- a/preppy/apps/merge_items.py +++ b/preppy/apps/merge_items.py @@ -1,11 +1,28 @@ +"""Merge separately-built catalogs into one. + +Two input shapes, auto-detected: + +* **output directories** (or their ``index.json``) — each per-object + subdirectory is copied into the output directory and the ``index.json`` + ``objects[]`` lists are merged into one. Object ``id`` is the merge key. +* **legacy ``items.json`` files** — the deprecated Voyager document list, merged + and de-duplicated in place (no assets involved). +""" + import argparse import json +import shutil from collections import OrderedDict from operator import itemgetter -from pathlib import Path +from pathlib import Path, PurePosixPath from natsort import natsorted +from preppy import manifest + +#: Stable name of the top-level archive listing in an output directory. +INDEX_NAME = 'index.json' + def merge_document_lists(a, b): """Merge two lists of documents""" @@ -66,25 +83,119 @@ def get_num_docs(data): return num_docs, num_groups -def main(): - parser = argparse.ArgumentParser(description='Merge items.json files.') - parser.add_argument('items', metavar='FILE', type=str, nargs='+', - help='List of items.json files') - parser.add_argument('-o', '--output-file', metavar='FILE', type=str, - default='merged.json', - help='Output path for merged items list') - parser.add_argument('--sort', default=True, - action=argparse.BooleanOptionalAction, - help='If enabled, sort the merged items file') - parser.add_argument('--sort-key', default='title', type=str, - choices=['title', 'document'], - help='Key to use when sorting the merged items file') - parser.add_argument('--merge-duplicates', default=True, - action=argparse.BooleanOptionalAction, - help='If enabled, merge duplicate entries ' - '(title and document match)') - args = parser.parse_args() +def is_index_input(path): + """True if ``path`` looks like an output directory (or its index.json).""" + path = Path(path) + return path.is_dir() or path.name == INDEX_NAME + + +def load_index(path): + """Load an output directory's listing; return ``(root_dir, objects)``. + + ``path`` is either the directory holding ``index.json`` or the file itself; + ``root_dir`` is the directory the entries' ``manifest``/``thumb`` URIs are + relative to. + """ + path = Path(path) + index = path / INDEX_NAME if path.is_dir() else path + if not index.is_file(): + raise FileNotFoundError(f"no '{INDEX_NAME}' in '{path}'") + + with index.open(encoding='utf8') as f: + data = json.load(f) + objects = data.get('objects') if isinstance(data, dict) else None + if not isinstance(objects, list): + raise ValueError(f"'{index}' is not an index.json (no 'objects' list)") + return index.parent, objects + + +def object_subdir(entry): + """The per-object subdirectory an index entry lives in.""" + parts = PurePosixPath(entry['manifest']).parts + if len(parts) < 2: + raise ValueError( + f"object '{entry.get('id')}' manifest '{entry['manifest']}' is not " + f'in a per-object subdirectory; nothing to merge') + return parts[0] + + +def plan_merge(sources, *, overwrite=False): + """Resolve sources to a list of ``(src_dir, subdir_name, entry)`` to copy. + + Duplicate object ``id``s across sources are an error unless ``overwrite``, + in which case the last source wins. Two different objects claiming the same + subdirectory is always an error — one would clobber the other's assets. + """ + planned = OrderedDict() + claimed = {} + for source in sources: + root, objects = load_index(source) + for entry in objects: + oid = entry.get('id') + if not oid: + raise ValueError(f"'{root}' has an index entry without an 'id'") + subdir = object_subdir(entry) + + owner = claimed.get(subdir) + if owner is not None and owner != oid: + raise ValueError( + f"objects '{owner}' and '{oid}' both use subdirectory " + f"'{subdir}'") + if oid in planned and not overwrite: + raise ValueError( + f"duplicate object '{oid}' in '{root}'; pass --overwrite " + f'to let the last source win') + claimed[subdir] = oid + planned[oid] = (root / subdir, subdir, entry) + return list(planned.values()) + + +def copy_object_dir(src, dst): + """Copy an object's asset directory, replacing any existing one.""" + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(src, dst) + + +def merge_output_dirs(sources, out_dir, *, overwrite=False, sort_key=None): + """Merge output directories into ``out_dir``; return the merged entries. + + Copies every object's subdirectory, then writes the combined + ``index.json``. The plan is resolved up front, so a collision fails before + anything is copied. + """ + out_dir = Path(out_dir) + planned = plan_merge(sources, overwrite=overwrite) + + out_dir.mkdir(parents=True, exist_ok=True) + entries = [] + for src, subdir, entry in planned: + if not src.is_dir(): + raise FileNotFoundError(f"missing object directory '{src}'") + copy_object_dir(src, out_dir / subdir) + entries.append(entry) + + if sort_key is not None: + entries = natsorted( + entries, key=lambda e: str(e.get(sort_key) or e.get('id') or '')) + + manifest.write_json(manifest.build_index(entries), out_dir / INDEX_NAME) + return entries + + +def merge_indexes(args): + """``main`` handler for the output-directory merge.""" + out_dir = Path(args.output_file or 'merged') + sort_key = args.sort_key if args.sort else None + entries = merge_output_dirs(args.items, out_dir, + overwrite=args.overwrite, sort_key=sort_key) + print(f'Merged {len(args.items)} director(ies): ' + f'{len(entries)} object(s) -> \'{out_dir}\'') + + +def merge_legacy_items(args): + """``main`` handler for the deprecated items.json merge.""" # Load everything into one list data = [] for i in args.items: @@ -114,11 +225,51 @@ def main(): print(f'Result: {num_groups} group(s), {num_docs} documents(s)') # Save the output - print(f'Writing to \'{args.output_file}\'') - with Path(args.output_file).open('w', encoding='utf8') as of: + output = args.output_file or 'merged.json' + print(f'Writing to \'{output}\'') + with Path(output).open('w', encoding='utf8') as of: json.dump(data, of, indent=2) print('Done') +def main(): + parser = argparse.ArgumentParser( + description='Merge preppy output directories (or legacy items.json ' + 'files) into one catalog.') + parser.add_argument('items', metavar='PATH', type=str, nargs='+', + help='Output directories (or their index.json), or ' + 'legacy items.json files') + parser.add_argument('-o', '--output-file', metavar='PATH', type=str, + default=None, + help='Output directory for a directory merge (default ' + '\'merged\'), else the merged items list ' + '(default \'merged.json\')') + parser.add_argument('--sort', default=True, + action=argparse.BooleanOptionalAction, + help='If enabled, sort the merged entries') + parser.add_argument('--sort-key', default='title', type=str, + choices=['title', 'id', 'document'], + help='Key to use when sorting the merged entries') + parser.add_argument('--overwrite', default=False, action='store_true', + help='Directory merge: let the last source win when ' + 'two sources provide the same object id') + parser.add_argument('--merge-duplicates', default=True, + action=argparse.BooleanOptionalAction, + help='items.json merge: merge duplicate entries ' + '(title and document match)') + args = parser.parse_args() + + index_inputs = [is_index_input(i) for i in args.items] + if all(index_inputs): + try: + merge_indexes(args) + except (ValueError, OSError) as e: + parser.error(str(e)) + elif any(index_inputs): + parser.error('cannot mix output directories and items.json files') + else: + merge_legacy_items(args) + + if __name__ == '__main__': main() diff --git a/tests/test_merge_items.py b/tests/test_merge_items.py new file mode 100644 index 0000000..f460f2d --- /dev/null +++ b/tests/test_merge_items.py @@ -0,0 +1,134 @@ +"""Unit tests for the catalog merge tool. The directory merge moves real assets +around, so the collision rules (duplicate ids, shared subdirectories) and the +up-front plan — nothing is copied until the whole merge is known to be valid — +are what's covered here. +""" + +import json + +import pytest + +from preppy.apps import merge_items + + +def make_output_dir(root, object_id, title=None, subdir=None): + """Build a minimal preppy output directory with one object.""" + subdir = subdir or object_id + obj_dir = root / subdir + obj_dir.mkdir(parents=True) + (obj_dir / 'manifest.json').write_text(json.dumps({'id': object_id})) + (obj_dir / f'{subdir}_rgb.abc12345.glb').write_bytes(b'glb') + entry = {'id': object_id, 'title': title or object_id, + 'manifest': f'{subdir}/manifest.json', + 'thumb': f'{subdir}/{subdir}_thumb.jpg'} + (root / 'index.json').write_text(json.dumps({'objects': [entry]})) + return root + + +def test_is_index_input(tmp_path): + d = tmp_path / 'out' + d.mkdir() + assert merge_items.is_index_input(d) + assert merge_items.is_index_input(d / 'index.json') + assert not merge_items.is_index_input(tmp_path / 'items.json') + + +def test_load_index_accepts_dir_or_file(tmp_path): + make_output_dir(tmp_path / 'a', 'OBJ') + root, objects = merge_items.load_index(tmp_path / 'a') + assert root == tmp_path / 'a' + assert objects[0]['id'] == 'OBJ' + # The index.json itself resolves to the same root. + assert merge_items.load_index(tmp_path / 'a' / 'index.json')[0] == root + + +def test_load_index_rejects_missing_and_malformed(tmp_path): + (tmp_path / 'empty').mkdir() + with pytest.raises(FileNotFoundError, match='index.json'): + merge_items.load_index(tmp_path / 'empty') + + bad = tmp_path / 'bad' + bad.mkdir() + (bad / 'index.json').write_text(json.dumps([{'id': 'OBJ'}])) # a list + with pytest.raises(ValueError, match='not an index.json'): + merge_items.load_index(bad) + + +def test_object_subdir_requires_a_subdirectory(): + assert merge_items.object_subdir({'manifest': 'OBJ/manifest.json'}) == 'OBJ' + with pytest.raises(ValueError, match='per-object subdirectory'): + merge_items.object_subdir({'id': 'OBJ', 'manifest': 'manifest.json'}) + + +def test_merge_output_dirs_copies_assets_and_merges_index(tmp_path): + a = make_output_dir(tmp_path / 'a', 'PHerc10Cr1', 'P.Herc. 10') + b = make_output_dir(tmp_path / 'b', 'PHerc2Cr1', 'P.Herc. 2') + out = tmp_path / 'merged' + + entries = merge_items.merge_output_dirs([a, b], out, sort_key='title') + + # Natural sort: 2 before 10. + assert [e['id'] for e in entries] == ['PHerc2Cr1', 'PHerc10Cr1'] + index = json.loads((out / 'index.json').read_text()) + assert [o['id'] for o in index['objects']] == ['PHerc2Cr1', 'PHerc10Cr1'] + # Assets came along, and the relative uris still resolve. + for entry in entries: + assert (out / entry['manifest']).is_file() + assert list((out / entry['id']).glob('*.glb')) + + +def test_merge_output_dirs_unsorted_keeps_source_order(tmp_path): + a = make_output_dir(tmp_path / 'a', 'B') + b = make_output_dir(tmp_path / 'b', 'A') + entries = merge_items.merge_output_dirs([a, b], tmp_path / 'out') + assert [e['id'] for e in entries] == ['B', 'A'] + + +def test_merge_output_dirs_refuses_duplicate_ids(tmp_path): + a = make_output_dir(tmp_path / 'a', 'OBJ') + b = make_output_dir(tmp_path / 'b', 'OBJ') + out = tmp_path / 'merged' + with pytest.raises(ValueError, match='duplicate object'): + merge_items.merge_output_dirs([a, b], out) + assert not out.exists() # planned up front: nothing was copied + + +def test_merge_output_dirs_overwrite_takes_last_source(tmp_path): + a = make_output_dir(tmp_path / 'a', 'OBJ', 'Old') + b = make_output_dir(tmp_path / 'b', 'OBJ', 'New') + (b / 'OBJ' / 'OBJ_rgb.abc12345.glb').write_bytes(b'newer') + out = tmp_path / 'merged' + + entries = merge_items.merge_output_dirs([a, b], out, overwrite=True) + + assert len(entries) == 1 and entries[0]['title'] == 'New' + assert (out / 'OBJ' / 'OBJ_rgb.abc12345.glb').read_bytes() == b'newer' + + +def test_merge_output_dirs_refuses_shared_subdir(tmp_path): + a = make_output_dir(tmp_path / 'a', 'ONE', subdir='shared') + b = make_output_dir(tmp_path / 'b', 'TWO', subdir='shared') + with pytest.raises(ValueError, match='both use subdirectory'): + merge_items.merge_output_dirs([a, b], tmp_path / 'merged') + + +def test_merge_output_dirs_replaces_stale_object_dir(tmp_path): + a = make_output_dir(tmp_path / 'a', 'OBJ') + out = tmp_path / 'merged' + stale = out / 'OBJ' + stale.mkdir(parents=True) + (stale / 'OBJ_rgb.deadbeef.glb').write_bytes(b'old generation') + + merge_items.merge_output_dirs([a], out) + + assert not (stale / 'OBJ_rgb.deadbeef.glb').exists() + assert (stale / 'OBJ_rgb.abc12345.glb').is_file() + + +def test_merge_duplicates_legacy_items(): + data = [{'title': 'A', 'document': 'a.json'}, + {'title': 'A', 'document': 'a.json'}, + {'title': 'G', 'subitems': [{'title': 'S', 'document': 's.json'}]}, + {'title': 'G', 'subitems': [{'title': 'T', 'document': 't.json'}]}] + merged = merge_items.merge_duplicates(data, 'title') + assert merge_items.get_num_docs(merged) == (3, 1) From ce072cd3c7ac472975445a4f284a9a45e9417fd3 Mon Sep 17 00:00:00 2001 From: Seth Parker Date: Sun, 20 Sep 2026 17:06:26 -0400 Subject: [PATCH 2/3] manifest: Carry inventory + sort into index.json entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An index entry was {id, title, manifest, thumb}, which is not enough to build a grouped, ordered host listing — the website's two-level index (P.Herc. group, then tray) had to read each webify dataset's metadata.json for the group title and sort key instead of the catalog. Copy the object's `inventory` (already the group title in practice) into its index entry, and add an optional object-level `sort` ordering key (index-only; not copied into the manifest). Both are omitted when unset. Also note at build_index() that index.json's shape has a second producer outside this repo: acquisition-workflow's pipeline/pipeline_deploy.py writes it from its own deploy state, since it can't import preppy. A change to the entry shape has to be mirrored there, and nothing fails loudly if it isn't — the viewer just stops finding objects. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 4 ++-- README.md | 7 ++++++- docs/implementation-plan.md | 2 ++ preppy/apps/file_prep.py | 3 ++- preppy/manifest.py | 28 ++++++++++++++++++++++++---- templates/prep-models.schema.json | 6 +++++- tests/test_integration.py | 4 ++++ tests/test_manifest.py | 11 +++++++++++ 8 files changed, 56 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 707c3a0..5a6c0c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,7 @@ Console entrypoints in `preppy/apps/` are thin argparse CLIs over the library mo - **`assemble.py`** — `embed()` runs the bundled Node helper to swap each material's baseColorTexture for its KTX2 (`KHR_texture_basisu`), preserving `EXT_meshopt_compression` (only if the meshopt encoder is registered — F3) and `KHR_texture_transform`. - **`preview.py`** — `render_preview()` renders a **proxy model preview** for the default-variant thumbnail (trimesh loads the OBJ + computes normals; pyrender renders one offscreen 3/4-view frame). A `FilePathResolver` subclass swaps the OBJ's `map_Kd` names for the already-normalized PNGs, so the raw (possibly gigapixel) sources are never decoded. It renders the OBJ, **not** the delivered meshopt/KTX2 glb (no offline renderer reads those) — recognizable, not pixel-identical. Raises `PreviewUnavailable` when the toolchain/GL backend is missing so the orchestrator falls back to the texture crop. Gotcha: these OBJs mix `f v/vt` and bare `f v` faces, so trimesh's "mixed data" fallback drops UVs for the whole mesh (and pyrender then compile-fails on a shader sampling a compiled-out `uv_0`); `_recover_uv_by_index` rebuilds `uv[i]=vt[i]` from the file (validated: one `vt` per vertex, matching `v/vt` indices, `process=False` keeps file order), else the model renders untextured. - **`cache.py`** — content hash over **inputs + config + tool versions** (never the output glb — basis encoding is non-deterministic), `hashed_name()`, and `prune()`. -- **`manifest.py`** (replaced `voyager.py`) — pure builders for the per-object `manifest.json` (flat `variants[] {id,label,uri,default}` + object metadata, `units:"cm"`, per-variant overrides) and the optional `index.json`. +- **`manifest.py`** (replaced `voyager.py`) — pure builders for the per-object `manifest.json` (flat `variants[] {id,label,uri,default}` + object metadata, `units:"cm"`, per-variant overrides) and the optional `index.json` (entries carry `inventory`/`sort` when set, so a host can group + order a catalog without opening every manifest). **`index.json`'s shape has a second producer outside this repo** — `acquisition-workflow`'s `pipeline/pipeline_deploy.py` writes it from its own deploy state (it can't import preppy, which lives in a container); a change to `build_index`/`index_entry` has to be mirrored there and nothing fails loudly if it isn't. - **`apps/file_prep.py`** — the orchestrator. Per object, per **variant** (no grouping): resolve texture(s) → normalize + encode each → gltfpack → optional Hausdorff gate → embed all by name → one self-contained glb → manifest entry. Emits `manifest.json` per object + a top-level `index.json`. The default-variant thumbnail is a rendered model preview (`preview.render_preview`, `_render_thumbnail` helper) that falls back to a texture crop (`--thumbnail-mode texture`, or automatically when the render toolchain is unavailable). - **`apps/merge_items.py`** — merges separately-built catalogs. Input shape is auto-detected: output directories (or their `index.json`) are merged by copying each per-object subdirectory into `-o` and combining the `index.json` `objects[]` (merge key is object `id`; collisions are planned up front so nothing is copied before the whole merge validates). Plain `.json` inputs take the deprecated `items.json` document-list path. - **`convert.py` + `apps/obj_to_glb.py`** — the deprecated legacy OBJ→Draco-GLB path (kept until removed). @@ -56,7 +56,7 @@ Console entrypoints in `preppy/apps/` are thin argparse CLIs over the library mo The `preppy` input JSON is a flat array of **objects**, validated by `templates/prep-models.schema.json` (examples: `prep-models-example.json`, `mvs-example.json`): -- An **object** needs `id`, `title`, and a `variants` array. Optional `prefix` (output folder/file prefix; defaults to `id`), `titles`, `inventory`, `description`, `credit`, `date`, `units` (default `cm`), `nodataFill`. +- An **object** needs `id`, `title`, and a `variants` array. Optional `prefix` (output folder/file prefix; defaults to `id`), `titles`, `inventory`, `description`, `credit`, `date`, `units` (default `cm`), `nodataFill`, `sort` (index-only ordering key; not copied into the manifest). - A **variant** needs `suffix` (stable key: names the file + is the manifest variant `id`) and `obj`. Optional `label`, `default`, `nodataFill` (resolved variant ?? object ?? CLI), and per-variant `credit`/`date`/`method`/`description`. Textures are resolved transitively from the OBJ's `map_Kd`. Relative `obj` paths resolve against `--data-root` (default CWD), not the config file's location. ### Output layout diff --git a/README.md b/README.md index f8d0f13..0de8a06 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,12 @@ out/ Asset filenames carry an inputs+config content hash by default (served `immutable`); `manifest.json` / `index.json` keep stable names and hold the -current hashed URIs. Useful flags: +current hashed URIs. Each `index.json` entry is `{id, title, manifest}` plus +the object's `inventory` and `sort` when the config sets them and `thumb` when +one was built — enough for a host to build a grouped, ordered listing from the +catalog alone, without opening every manifest. + +Useful flags: | Flag | Effect | | --- | --- | diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 1130e55..e374dad 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -115,6 +115,8 @@ flat listing for host pages only. // index.json (optional — host archive UI only) { "objects": [ { "id":"PHerc1061Cr05", "title":"P.Herc. 1061, Cr. 5", + "inventory":"P.Herc. 1061", // optional: group key for a host listing + "sort":"1061-05", // optional: explicit ordering key "manifest":"PHerc1061Cr05/manifest.json", "thumb":"PHerc1061Cr05/PHerc1061Cr05_thumb.jpg" } ] } diff --git a/preppy/apps/file_prep.py b/preppy/apps/file_prep.py index 2ff7f67..4bfd030 100644 --- a/preppy/apps/file_prep.py +++ b/preppy/apps/file_prep.py @@ -275,7 +275,8 @@ def process_object(object_cfg: Mapping, *, data_root: Path, out_dir: Path, return manifest.index_entry( object_id, man.get('title', object_id), - manifest_uri=f'{prefix}/manifest.json', thumb=thumb_rel) + manifest_uri=f'{prefix}/manifest.json', thumb=thumb_rel, + inventory=man.get('inventory'), sort=object_cfg.get('sort')) def _build_parser() -> argparse.ArgumentParser: diff --git a/preppy/manifest.py b/preppy/manifest.py index 34a55b9..0c2051c 100644 --- a/preppy/manifest.py +++ b/preppy/manifest.py @@ -111,16 +111,36 @@ def build_manifest(object_cfg: Mapping, def index_entry(object_id: str, title: str, manifest_uri: str, *, - thumb: Optional[str] = None) -> Dict: - """Build one ``index.json`` ``objects[]`` entry (host archive UI only).""" - entry: Dict = {'id': object_id, 'title': title, 'manifest': manifest_uri} + thumb: Optional[str] = None, + inventory: Optional[str] = None, + sort: Optional[str] = None) -> Dict: + """Build one ``index.json`` ``objects[]`` entry (host archive UI only). + + ``inventory`` (the object's group / shelf number) and ``sort`` (an explicit + ordering key) are omitted when unset. They carry enough object metadata for + a host to build a grouped, ordered listing from a catalog alone, instead of + opening every manifest. + """ + entry: Dict = {'id': object_id, 'title': title} + if inventory is not None: + entry['inventory'] = inventory + if sort is not None: + entry['sort'] = sort + entry['manifest'] = manifest_uri if thumb is not None: entry['thumb'] = thumb return entry def build_index(objects: Sequence[Mapping]) -> Dict: - """Wrap ``index_entry`` results into the optional top-level ``index.json``.""" + """Wrap ``index_entry`` results into the optional top-level ``index.json``. + + Schema note: ``index.json`` has a second producer outside this repo. + ``acquisition-workflow``'s ``pipeline/pipeline_deploy.py`` writes the same + shape from its own deploy state (it cannot import preppy, which lives in a + container). A change to the entry shape has to be mirrored there, and + nothing fails loudly if it is not — the viewer just stops finding objects. + """ return {'objects': list(objects)} diff --git a/templates/prep-models.schema.json b/templates/prep-models.schema.json index 2fa3354..f17d8ab 100644 --- a/templates/prep-models.schema.json +++ b/templates/prep-models.schema.json @@ -53,7 +53,11 @@ "minProperties": 1 }, "inventory": { - "description": "Inventory / shelf number.", + "description": "Inventory / shelf number. Also copied into the object's index.json entry, where a host UI can group by it.", + "type": "string" + }, + "sort": { + "description": "Optional ordering key for this object in index.json (host listings only; not copied into the manifest).", "type": "string" }, "description": { "$ref": "#/$defs/localizedString" }, diff --git a/tests/test_integration.py b/tests/test_integration.py index 8d6bbfb..01cc913 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -68,6 +68,7 @@ def sample(tmp_path): config = [{ 'id': 'MVS', 'title': 'MVS Sample', 'inventory': 'MVS-1', + 'sort': 'MVS-0001', 'credit': '(c) UK', 'date': '2026-07-08', 'units': 'cm', 'variants': [ {'suffix': 'rgb', 'label': 'RGB', 'obj': 'src/mesh.obj', @@ -103,6 +104,9 @@ def test_pipeline_emits_manifest_and_assets(sample): index = json.loads((out_dir / 'index.json').read_text()) assert [o['id'] for o in index['objects']] == ['MVS'] assert index['objects'][0]['thumb'] == 'MVS/MVS_thumb.jpg' + # Group/order metadata rides along so a host can build a grouped listing. + assert index['objects'][0]['inventory'] == 'MVS-1' + assert index['objects'][0]['sort'] == 'MVS-0001' assert (out_dir / 'MVS' / 'MVS_thumb.jpg').is_file() # per-object manifest shape. diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 6b320df..506337e 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -97,6 +97,17 @@ def test_index_entry_and_build_index(): assert idx == {'objects': [e]} +def test_index_entry_group_metadata(): + """inventory/sort let a host group and order a catalog without opening + every manifest; both are omitted when unset.""" + e = manifest.index_entry('OBJ', 'Cr. 5', 'OBJ/manifest.json', + inventory='P.Herc. 1061', sort='0005') + assert e['inventory'] == 'P.Herc. 1061' + assert e['sort'] == '0005' + plain = manifest.index_entry('OBJ', 'Cr. 5', 'OBJ/manifest.json') + assert 'inventory' not in plain and 'sort' not in plain + + def test_write_json_roundtrip(tmp_path): data = {'id': 'OBJ', 'variants': [{'id': 'rgb'}]} out = manifest.write_json(data, tmp_path / 'sub' / 'manifest.json') From 40827f396e0c4d1eb72d6fe0367443dc41f2ddbb Mon Sep 17 00:00:00 2001 From: Seth Parker Date: Sun, 20 Sep 2026 17:06:43 -0400 Subject: [PATCH 3/3] merge-items: Incremental --update and --index-only merges The directory merge required every source on every invocation: it wrote a fresh index.json from exactly the sources it was handed and rmtree+copytree'd each one, so adding three objects to a catalog meant recopying the whole corpus. --update CATALOG seeds the merge plan from an existing catalog's index.json and rewrites it in place, copying only the named sources' object directories and leaving every other object untouched. A base object a source redeclares is replaced without --overwrite (that is what an update is); duplicate ids among the sources are still an error, and the subdirectory-collision check now covers the base's claims too. --index-only merges the listings and copies no assets, so a catalog whose assets are not reachable from here (a remote host) can still be updated from a copy of its index.json. Neither applies to the legacy items.json path; the CLI says so rather than quietly ignoring them. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 3 +- README.md | 25 ++++++++++- docs/implementation-plan.md | 3 +- preppy/apps/merge_items.py | 87 +++++++++++++++++++++++++++++-------- tests/test_merge_items.py | 85 ++++++++++++++++++++++++++++++++++++ 5 files changed, 181 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5a6c0c9..836107d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,6 +32,7 @@ preppy -i config.json -o out/ # batch: variants → self-contained glb preppy-check-tools # report external toolchain status preppy-obj2glb -i mesh.obj -o mesh.glb # LEGACY single OBJ → Draco GLB (deprecated) preppy-merge-items out1/ out2/ -o merged/ # merge output dirs (assets + index.json) +preppy-merge-items new/ --update catalog/ # incremental merge into an existing catalog preppy-merge-items a.json b.json -o merged.json # LEGACY items.json merge (deprecated) ``` @@ -49,7 +50,7 @@ Console entrypoints in `preppy/apps/` are thin argparse CLIs over the library mo - **`cache.py`** — content hash over **inputs + config + tool versions** (never the output glb — basis encoding is non-deterministic), `hashed_name()`, and `prune()`. - **`manifest.py`** (replaced `voyager.py`) — pure builders for the per-object `manifest.json` (flat `variants[] {id,label,uri,default}` + object metadata, `units:"cm"`, per-variant overrides) and the optional `index.json` (entries carry `inventory`/`sort` when set, so a host can group + order a catalog without opening every manifest). **`index.json`'s shape has a second producer outside this repo** — `acquisition-workflow`'s `pipeline/pipeline_deploy.py` writes it from its own deploy state (it can't import preppy, which lives in a container); a change to `build_index`/`index_entry` has to be mirrored there and nothing fails loudly if it isn't. - **`apps/file_prep.py`** — the orchestrator. Per object, per **variant** (no grouping): resolve texture(s) → normalize + encode each → gltfpack → optional Hausdorff gate → embed all by name → one self-contained glb → manifest entry. Emits `manifest.json` per object + a top-level `index.json`. The default-variant thumbnail is a rendered model preview (`preview.render_preview`, `_render_thumbnail` helper) that falls back to a texture crop (`--thumbnail-mode texture`, or automatically when the render toolchain is unavailable). -- **`apps/merge_items.py`** — merges separately-built catalogs. Input shape is auto-detected: output directories (or their `index.json`) are merged by copying each per-object subdirectory into `-o` and combining the `index.json` `objects[]` (merge key is object `id`; collisions are planned up front so nothing is copied before the whole merge validates). Plain `.json` inputs take the deprecated `items.json` document-list path. +- **`apps/merge_items.py`** — merges separately-built catalogs. Input shape is auto-detected: output directories (or their `index.json`) are merged by copying each per-object subdirectory into `-o` and combining the `index.json` `objects[]` (merge key is object `id`; collisions are planned up front so nothing is copied before the whole merge validates). `--update CATALOG` seeds the plan from an existing catalog and rewrites it in place, copying only the named sources' objects (a base object a source redeclares is replaced without `--overwrite`); `--index-only` merges the listings and copies nothing, for a catalog whose assets aren't reachable. Plain `.json` inputs take the deprecated `items.json` document-list path. - **`convert.py` + `apps/obj_to_glb.py`** — the deprecated legacy OBJ→Draco-GLB path (kept until removed). ### Input config format diff --git a/README.md b/README.md index 0de8a06..7d46444 100644 --- a/README.md +++ b/README.md @@ -130,10 +130,31 @@ by keeping each object's subdirectory name. Duplicate ids across sources are an error — pass `--overwrite` to let the last source win, which is how a rebuilt object replaces its older assets in place. The whole merge is validated before anything is copied, and `--no-sort` keeps source order instead of sorting by -title. +title (`--sort-key` also takes `id`, `sort` or `inventory`). + +To add a few objects to a catalog that already exists, `--update` merges them in +place instead of rebuilding the whole thing — only the named sources' object +directories are replaced, and every other object is left untouched: + +```shell +preppy-merge-items rebuilt/ new/ --update catalog/ +``` + +An object already in the catalog is replaced without `--overwrite` (that is what +an update is); duplicate ids *among the sources* are still an error. The catalog +is rewritten in place, so `-o` is unused. + +`--index-only` merges the listings and copies no assets, for a catalog whose +assets live somewhere this machine cannot reach — point it at a copy of the +remote `index.json`: + +```shell +preppy-merge-items new/ --update ./catalog-index/index.json --index-only +``` Passing `.json` files instead of directories runs the deprecated `items.json` -document-list merge (`--merge-duplicates`). +document-list merge (`--merge-duplicates`); `--update` and `--index-only` do not +apply to it. ### Hosting / caching diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index e374dad..6fd4e72 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -164,7 +164,8 @@ mogrify — cheap, no offscreen GL. (Loose end: rendered 3D thumbs later.) Keep console-script entry; add `--ktx2-mode {uastc,etc1s}`, `--decimate-error`, `--no-decimate`, keep `--uri`, `--keep-tmp`. `merge_items` keeps its name (no rename to `merge_index`); it gained the output-directory merge alongside the -legacy `items.json` one. +legacy `items.json` one, plus `--update CATALOG` (merge in place, touching only +the named objects) and `--index-only` (listings without assets). --- diff --git a/preppy/apps/merge_items.py b/preppy/apps/merge_items.py index 92152e8..0425c62 100644 --- a/preppy/apps/merge_items.py +++ b/preppy/apps/merge_items.py @@ -5,6 +5,8 @@ * **output directories** (or their ``index.json``) — each per-object subdirectory is copied into the output directory and the ``index.json`` ``objects[]`` lists are merged into one. Object ``id`` is the merge key. + ``--update`` merges into an existing catalog in place (only the named + sources move); ``--index-only`` merges the listings and copies nothing. * **legacy ``items.json`` files** — the deprecated Voyager document list, merged and de-duplicated in place (no assets involved). """ @@ -89,6 +91,15 @@ def is_index_input(path): return path.is_dir() or path.name == INDEX_NAME +def index_root(path): + """The directory an index's ``manifest``/``thumb`` URIs resolve against. + + Accepts either the output directory or its ``index.json``. + """ + path = Path(path) + return path.parent if path.name == INDEX_NAME else path + + def load_index(path): """Load an output directory's listing; return ``(root_dir, objects)``. @@ -96,10 +107,10 @@ def load_index(path): ``root_dir`` is the directory the entries' ``manifest``/``thumb`` URIs are relative to. """ - path = Path(path) - index = path / INDEX_NAME if path.is_dir() else path + root = index_root(path) + index = root / INDEX_NAME if not index.is_file(): - raise FileNotFoundError(f"no '{INDEX_NAME}' in '{path}'") + raise FileNotFoundError(f"no '{INDEX_NAME}' in '{root}'") with index.open(encoding='utf8') as f: data = json.load(f) @@ -119,16 +130,25 @@ def object_subdir(entry): return parts[0] -def plan_merge(sources, *, overwrite=False): +def plan_merge(sources, *, overwrite=False, base=None): """Resolve sources to a list of ``(src_dir, subdir_name, entry)`` to copy. Duplicate object ``id``s across sources are an error unless ``overwrite``, in which case the last source wins. Two different objects claiming the same subdirectory is always an error — one would clobber the other's assets. + + ``base`` is an existing catalog seeding the merge (an incremental update): + its objects are planned with a ``src_dir`` of ``None`` — already in place, + nothing to copy — and keep their position in the listing. A source + redeclaring one replaces it without ``overwrite``; that is the point. """ planned = OrderedDict() claimed = {} - for source in sources: + replaceable = set() + + stages = [(base, True)] if base is not None else [] + stages += [(source, False) for source in sources] + for source, is_base in stages: root, objects = load_index(source) for entry in objects: oid = entry.get('id') @@ -141,13 +161,17 @@ def plan_merge(sources, *, overwrite=False): raise ValueError( f"objects '{owner}' and '{oid}' both use subdirectory " f"'{subdir}'") - if oid in planned and not overwrite: + if oid in planned and oid not in replaceable and not overwrite: raise ValueError( f"duplicate object '{oid}' in '{root}'; pass --overwrite " f'to let the last source win') claimed[subdir] = oid - planned[oid] = (root / subdir, subdir, entry) + planned[oid] = (None if is_base else root / subdir, subdir, entry) + if is_base: + replaceable.add(oid) + else: + replaceable.discard(oid) return list(planned.values()) @@ -158,22 +182,29 @@ def copy_object_dir(src, dst): shutil.copytree(src, dst) -def merge_output_dirs(sources, out_dir, *, overwrite=False, sort_key=None): +def merge_output_dirs(sources, out_dir, *, overwrite=False, sort_key=None, + base=None, index_only=False): """Merge output directories into ``out_dir``; return the merged entries. Copies every object's subdirectory, then writes the combined ``index.json``. The plan is resolved up front, so a collision fails before anything is copied. + + ``base`` (normally ``out_dir`` itself) makes this an incremental update: + only the sources' objects are copied and the rest of the catalog is left + alone. ``index_only`` merges the listings and copies nothing, for a catalog + whose assets are not reachable from here. """ out_dir = Path(out_dir) - planned = plan_merge(sources, overwrite=overwrite) + planned = plan_merge(sources, overwrite=overwrite, base=base) out_dir.mkdir(parents=True, exist_ok=True) entries = [] for src, subdir, entry in planned: - if not src.is_dir(): - raise FileNotFoundError(f"missing object directory '{src}'") - copy_object_dir(src, out_dir / subdir) + if src is not None and not index_only: + if not src.is_dir(): + raise FileNotFoundError(f"missing object directory '{src}'") + copy_object_dir(src, out_dir / subdir) entries.append(entry) if sort_key is not None: @@ -186,12 +217,15 @@ def merge_output_dirs(sources, out_dir, *, overwrite=False, sort_key=None): def merge_indexes(args): """``main`` handler for the output-directory merge.""" - out_dir = Path(args.output_file or 'merged') + base = args.update + out_dir = index_root(base) if base else Path(args.output_file or 'merged') sort_key = args.sort_key if args.sort else None - entries = merge_output_dirs(args.items, out_dir, - overwrite=args.overwrite, sort_key=sort_key) - print(f'Merged {len(args.items)} director(ies): ' - f'{len(entries)} object(s) -> \'{out_dir}\'') + entries = merge_output_dirs(args.items, out_dir, overwrite=args.overwrite, + sort_key=sort_key, base=base, + index_only=args.index_only) + target = out_dir / INDEX_NAME if args.index_only else out_dir + print(f'{"Updated" if base else "Merged"} {len(args.items)} source(s): ' + f'{len(entries)} object(s) -> \'{target}\'') def merge_legacy_items(args): @@ -248,17 +282,31 @@ def main(): action=argparse.BooleanOptionalAction, help='If enabled, sort the merged entries') parser.add_argument('--sort-key', default='title', type=str, - choices=['title', 'id', 'document'], + choices=['title', 'id', 'sort', 'inventory', + 'document'], help='Key to use when sorting the merged entries') parser.add_argument('--overwrite', default=False, action='store_true', help='Directory merge: let the last source win when ' 'two sources provide the same object id') + parser.add_argument('--update', metavar='CATALOG', type=str, default=None, + help='Directory merge: merge the sources into an ' + 'existing catalog (a directory or its ' + 'index.json) in place, replacing only their ' + 'objects and leaving the rest untouched') + parser.add_argument('--index-only', default=False, action='store_true', + help='Directory merge: merge the index.json listings ' + 'without copying any assets') parser.add_argument('--merge-duplicates', default=True, action=argparse.BooleanOptionalAction, help='items.json merge: merge duplicate entries ' '(title and document match)') args = parser.parse_args() + if args.update and args.output_file: + parser.error('--update rewrites the catalog in place; -o is unused') + + dir_only = (('--update', args.update), ('--index-only', args.index_only)) + dir_flags = [flag for flag, given in dir_only if given] index_inputs = [is_index_input(i) for i in args.items] if all(index_inputs): try: @@ -267,6 +315,9 @@ def main(): parser.error(str(e)) elif any(index_inputs): parser.error('cannot mix output directories and items.json files') + elif dir_flags: + parser.error(f'{", ".join(dir_flags)} only applies to an ' + f'output-directory merge') else: merge_legacy_items(args) diff --git a/tests/test_merge_items.py b/tests/test_merge_items.py index f460f2d..80f11d9 100644 --- a/tests/test_merge_items.py +++ b/tests/test_merge_items.py @@ -125,6 +125,91 @@ def test_merge_output_dirs_replaces_stale_object_dir(tmp_path): assert (stale / 'OBJ_rgb.abc12345.glb').is_file() +def test_index_root_accepts_dir_or_index_file(tmp_path): + assert merge_items.index_root(tmp_path / 'cat') == tmp_path / 'cat' + assert merge_items.index_root(tmp_path / 'cat' / 'index.json') == \ + tmp_path / 'cat' + + +def make_catalog(root, ids): + """Build a merged catalog holding one object per id.""" + sources = [make_output_dir(root / f'src_{oid}', oid) for oid in ids] + merge_items.merge_output_dirs(sources, root / 'catalog') + return root / 'catalog' + + +def test_update_replaces_only_the_named_object(tmp_path): + catalog = make_catalog(tmp_path, ['KEEP', 'STALE']) + (catalog / 'KEEP' / 'untouched.txt').write_text('still here') + new = make_output_dir(tmp_path / 'new', 'STALE', 'Rebuilt') + (new / 'STALE' / 'STALE_rgb.abc12345.glb').write_bytes(b'newer') + + entries = merge_items.merge_output_dirs([new], catalog, base=catalog) + + # The base object keeps its entry, its position, and its assets. + assert [e['id'] for e in entries] == ['KEEP', 'STALE'] + assert (catalog / 'KEEP' / 'untouched.txt').is_file() + # The redeclared object was replaced without --overwrite. + assert entries[1]['title'] == 'Rebuilt' + assert (catalog / 'STALE' / 'STALE_rgb.abc12345.glb').read_bytes() == \ + b'newer' + index = json.loads((catalog / 'index.json').read_text()) + assert [o['id'] for o in index['objects']] == ['KEEP', 'STALE'] + + +def test_update_appends_new_objects(tmp_path): + catalog = make_catalog(tmp_path, ['A']) + new = make_output_dir(tmp_path / 'new', 'B') + + entries = merge_items.merge_output_dirs([new], catalog, base=catalog) + + assert [e['id'] for e in entries] == ['A', 'B'] + assert (catalog / 'B' / 'manifest.json').is_file() + + +def test_update_still_refuses_duplicates_among_sources(tmp_path): + catalog = make_catalog(tmp_path, ['OBJ']) + a = make_output_dir(tmp_path / 'a', 'OBJ') + b = make_output_dir(tmp_path / 'b', 'OBJ') + with pytest.raises(ValueError, match='duplicate object'): + merge_items.merge_output_dirs([a, b], catalog, base=catalog) + + +def test_update_refuses_subdir_claimed_by_another_object(tmp_path): + catalog = make_catalog(tmp_path, ['ONE']) + other = make_output_dir(tmp_path / 'other', 'TWO', subdir='ONE') + with pytest.raises(ValueError, match='both use subdirectory'): + merge_items.merge_output_dirs([other], catalog, base=catalog) + + +def test_index_only_merges_listings_without_assets(tmp_path): + # A catalog whose assets are elsewhere (a remote host): only its index.json + # is reachable, and no source directory is copied. + remote = tmp_path / 'remote' + remote.mkdir() + (remote / 'index.json').write_text(json.dumps( + {'objects': [{'id': 'A', 'title': 'A', + 'manifest': 'A/manifest.json'}]})) + new = make_output_dir(tmp_path / 'new', 'B') + + entries = merge_items.merge_output_dirs([new], remote, base=remote, + index_only=True) + + assert [e['id'] for e in entries] == ['A', 'B'] + assert not (remote / 'B').exists() + index = json.loads((remote / 'index.json').read_text()) + assert [o['id'] for o in index['objects']] == ['A', 'B'] + + +def test_index_only_accepts_an_index_file_as_the_source(tmp_path): + a = make_output_dir(tmp_path / 'a', 'A') + out = tmp_path / 'listing' + entries = merge_items.merge_output_dirs([a / 'index.json'], out, + index_only=True) + assert [e['id'] for e in entries] == ['A'] + assert not (out / 'A').exists() + + def test_merge_duplicates_legacy_items(): data = [{'title': 'A', 'document': 'a.json'}, {'title': 'A', 'document': 'a.json'},