Fix DB symlink "File exists" failure and generate-db-symlinks.sh drift

code/ARAX/ARAXQuery/ARAX_database_manager.py

@@ -372,7 +372,11 @@ def _extract_tarball(self, tarball_path, debug=False):
             eprint(f"Looks like we have previously extracted: {resolved}")
 
     def symlink_database(self, symlink_path, target_path):
-        _run_cmd_in_shell_chk_status(f"ln -s {shlex.quote(target_path)} {shlex.quote(symlink_path)}")
+        # -sfn replaces an existing (possibly stale or dangling) symlink in place,
+        # so re-running the database manager is idempotent. Never clobber a real file.
+        if os.path.lexists(symlink_path) and not os.path.islink(symlink_path):
+            raise FileExistsError(f"{symlink_path} exists and is not a symlink; refusing to overwrite")
+        _run_cmd_in_shell_chk_status(f"ln -sfn {shlex.quote(target_path)} {shlex.quote(symlink_path)}")
 
     def rsync_database(self, remote_location, local_path, debug=False):
         verbose = ""

code/ARAX/test/test_ARAX_database_manager.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""
+Unit tests for ARAXDatabaseManager.symlink_database().
+
+The CLI path (`python3 ARAX_database_manager.py`, run by ITRB start_app.sh, the pytest
+workflow and deploy/preview/deploy.sh) used to die with "ln: ...: File exists" whenever a
+link was already present. These tests check that re-linking is idempotent, that a stale or
+dangling link is repointed, and that a real file is never clobbered. They only touch a
+pytest tmp_path, so they need no databases and no network access.
+
+Usage:
+  pytest -v test_ARAX_database_manager.py
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../ARAXQuery")
+from ARAX_database_manager import ARAXDatabaseManager  # noqa: E402
+
+
+@pytest.fixture
+def manager():
+    # symlink_database() uses no instance state; skip __init__ so the test does not
+    # need RTXConfiguration or create directories inside the RTX checkout.
+    return ARAXDatabaseManager.__new__(ARAXDatabaseManager)
+
+
+def test_symlink_database_is_idempotent(manager, tmp_path):
+    target = tmp_path / "central.sqlite"
+    target.write_text("db")
+    link = tmp_path / "local.sqlite"
+    manager.symlink_database(symlink_path=str(link), target_path=str(target))
+    manager.symlink_database(symlink_path=str(link), target_path=str(target))
+    assert os.readlink(link) == str(target)
+
+
+@pytest.mark.parametrize("old_target", ["old-central.sqlite", "does-not-exist.sqlite"])
+def test_symlink_database_repoints_stale_link(manager, tmp_path, old_target):
+    (tmp_path / "old-central.sqlite").write_text("old")
+    target = tmp_path / "central.sqlite"
+    target.write_text("new")
+    link = tmp_path / "local.sqlite"
+    os.symlink(tmp_path / old_target, link)
+    manager.symlink_database(symlink_path=str(link), target_path=str(target))
+    assert os.readlink(link) == str(target)
+
+
+def test_symlink_database_refuses_to_overwrite_real_file(manager, tmp_path):
+    target = tmp_path / "central.sqlite"
+    target.write_text("db")
+    real_file = tmp_path / "local.sqlite"
+    real_file.write_text("local copy")
+    with pytest.raises(FileExistsError):
+        manager.symlink_database(symlink_path=str(real_file), target_path=str(target))
+    assert not real_file.is_symlink()
+    assert real_file.read_text() == "local copy"

code/generate-db-symlinks.sh

@@ -1,26 +1,77 @@
 #!/usr/bin/env bash
 set -euo pipefail
 # Purpose: sets up symbolic links for ARAX database files
+# The databases and their file names are read from RTX/code/config_dbs.json
+# (the source of truth), so this script does not need editing when a
+# database version changes there.
 # Usage: it's a three-step process
-#   1. configure `DB_DIR` and `LATEST_KG2_VER` shell variabless
+#   1. configure the `DB_DIR` shell variable below (or export it); it must
+#      hold the databases laid out as on arax-databases.rtx.ai, e.g.
+#      ${DB_DIR}/tier0-20260621/curie_ngd_v1.1_tier0-20260621.sqlite
 #   2. `cd` to the directory just above the `RTX` root code directory
-#   3. `./generate-db-symlinks.sy`
-# When you run the pytest suite, don't forget to pass `--nodatabases`
-# so that the database manager won't attempt to download 200 GiB worth
-# of databases to your development machine.
+#   3. `RTX/code/generate-db-symlinks.sh`
+# `LATEST_TIER0_VER` defaults to the tier0 version of `tier0_sqlite` in
+# config_dbs.json. Set it to link a different tier0 build; databases that
+# config_dbs.json keeps in another directory (e.g. fda_approved_drugs) are
+# not affected by it.
+# The pytest suite only runs the database manager if you pass
+# `--withdatabases`, so by default it won't attempt to download 200 GiB
+# worth of databases to your development machine.
 
 # Stephen Ramsey, Oregon State University
 
-LATEST_TIER0_VER="tier0-20260621"
+DB_DIR="${DB_DIR:-/mnt/data/orangeboard/databases}"
 
-DB_DIR="/mnt/data/orangeboard/databases"
+CONFIG_DBS="RTX/code/config_dbs.json"
 
 if [[ ! -d "RTX" ]]; then
   echo "ERROR: Expected to find directory 'RTX' in the current working directory: $(pwd)" >&2
   echo "       Run this script from the directory one level above RTX." >&2
   exit 1
 fi
 
+# One "key<TAB>dir<TAB>file" line per entry under database_downloads, e.g.
+#   curie_ngd	tier0-20260621	curie_ngd_v1.1_tier0-20260621.sqlite
+db_entries="$(python3 -c '
+import json, sys
+with open(sys.argv[1]) as f:
+    downloads = json.load(f)["database_downloads"]
+for key, path in downloads.items():
+    print("\t".join([key] + path.split("/")[-2:]))
+' "$CONFIG_DBS")"
+
+config_tier0_ver="$(awk -F'\t' '$1 == "tier0_sqlite" {print $2}' <<< "$db_entries")"
+if [[ -z "$config_tier0_ver" ]]; then
+  echo "ERROR: No database_downloads.tier0_sqlite entry in $CONFIG_DBS" >&2
+  exit 1
+fi
+LATEST_TIER0_VER="${LATEST_TIER0_VER:-$config_tier0_ver}"
+
+# Where each database goes in the RTX tree. Must match `local_paths` in
+# ARAX_database_manager.py (and the Tier0 dir that ARAX_decorator.py reads).
+dest_dir_for() {
+  case "$1" in
+    cohd_database)            echo "RTX/code/ARAX/KnowledgeSources/COHD_local/data" ;;
+    curie_to_pmids|curie_ngd) echo "RTX/code/ARAX/KnowledgeSources/NormalizedGoogleDistance" ;;
+    kg2c_sqlite)              echo "RTX/code/ARAX/KnowledgeSources/KG2c" ;;
+    tier0_sqlite)             echo "RTX/code/ARAX/KnowledgeSources/Tier0" ;;
+    fda_approved_drugs)       echo "RTX/code/ARAX/KnowledgeSources" ;;
+    autocomplete)             echo "RTX/code/autocomplete" ;;
+    explainable_dtd_db)       echo "RTX/code/ARAX/KnowledgeSources/Prediction" ;;
+    *)                        return 1 ;;
+  esac
+}
+
+# Fail before creating any links if config_dbs.json has a database this
+# script does not know where to put.
+while IFS=$'\t' read -r key _ _; do
+  if ! dest_dir_for "$key" > /dev/null; then
+    echo "ERROR: $CONFIG_DBS has database '$key' but $0 has no destination for it." >&2
+    echo "       Add it to dest_dir_for(), matching ARAX_database_manager.py." >&2
+    exit 1
+  fi
+done <<< "$db_entries"
+
 link() {
   local src="$1"
   local dest="$2"
@@ -42,11 +93,10 @@ link() {
   ln -sfn "$src" "$dest"
 }
 
-link "${DB_DIR}/KG2.8.0/COHDdatabase_v1.0_KG2.8.0.db" "RTX/code/ARAX/KnowledgeSources/COHD_local/data/COHDdatabase_v1.0_KG2.8.0.db"
-
-link "${DB_DIR}/${LATEST_TIER0_VER}/ExplainableDTD_v1.0_${LATEST_TIER0_VER}-all_with_paths.db" "RTX/code/ARAX/KnowledgeSources/Prediction/ExplainableDTD_v1.0_${LATEST_TIER0_VER}-all_with_paths.db"
-link "${DB_DIR}/${LATEST_TIER0_VER}/curie_ngd_v1.0_${LATEST_TIER0_VER}.sqlite" "RTX/code/ARAX/KnowledgeSources/NormalizedGoogleDistance/curie_ngd_v1.0_${LATEST_TIER0_VER}.sqlite"
-link "${DB_DIR}/${LATEST_TIER0_VER}/curie_to_pmids_v1.0_${LATEST_TIER0_VER}.sqlite" "RTX/code/ARAX/KnowledgeSources/NormalizedGoogleDistance/curie_to_pmids_v1.0_${LATEST_TIER0_VER}.sqlite"
-link "${DB_DIR}/${LATEST_TIER0_VER}/fda_approved_drugs_v1.0.pickle" "RTX/code/ARAX/KnowledgeSources/fda_approved_drugs_v1.0.pickle"
-link "${DB_DIR}/${LATEST_TIER0_VER}/tier0-info-for-overlay_v1.0_${LATEST_TIER0_VER}.sqlite" "RTX/code/ARAX/KnowledgeSources/KG2c/tier0-info-for-overlay_v1.0_${LATEST_TIER0_VER}.sqlite"
-link "${DB_DIR}/${LATEST_TIER0_VER}/autocomplete_v1.0_${LATEST_TIER0_VER}.sqlite" "RTX/code/autocomplete/autocomplete_v1.0_${LATEST_TIER0_VER}.sqlite"
+while IFS=$'\t' read -r key dir file; do
+  if [[ "$dir" == "$config_tier0_ver" ]]; then
+    dir="$LATEST_TIER0_VER"
+    file="${file//$config_tier0_ver/$LATEST_TIER0_VER}"
+  fi
+  link "${DB_DIR}/${dir}/${file}" "$(dest_dir_for "$key")/${file}"
+done <<< "$db_entries"