member-console/scripts/agent-runner/run-audit.sh

#!/usr/bin/env bash
# Ralph-style audit loop: for each model, for each scope task, run a FRESH
# auditor container (clean context) and capture its findings. Sequential by
# design — concurrent opencode runs in one project deadlock, and it respects
# provider rate limits.
#
# Usage:
#   OUT=<dir> ./run-audit.sh                         # default models, all tasks
#   OUT=<dir> MODELS="deepseek/deepseek-v4-pro" ./run-audit.sh
#   OUT=<dir> MODELS="codex:gpt-6-astra" VARIANT=ultra ./run-audit.sh   # Codex CLI lane
#   OUT=<dir> ./run-audit.sh 02-authorization 05-csp-headers  # only these slices
#   OUT=<dir> MODELS="codex:gpt-6-astra" VARIANT=ultra TIMEOUT=14400 ./run-audit.sh open/bare
#   OUT=<dir> TIMEOUT=1800 ./run-audit.sh
#   OUT=<dir> VARIANT=high ./run-audit.sh   # reasoning effort; default max, "" to omit
#   OUT=<dir> QUOTA_POLL=600 QUOTA_WAIT=43200 ./run-audit.sh   # usage-limit resume cadence and ceiling
#
# A task is a slice under scope/ (the default set is all of them) or an open
# brief named as "open/<name>", which hands the model the whole repository and
# no questions. A MODELS entry is "<harness>:<model>"; no prefix means
# opencode. harness.sh holds what differs per harness: the id syntax, the
# credential each lane mounts, how VARIANT is passed, and how a task resumes
# after the plan's usage window runs out.
#
# OUT names the directory that receives findings. Keep it outside this one:
# this directory is tracked, and a run's output is not.
set -euo pipefail
cd "$(dirname "$0")"
HERE="$(pwd)"
PROJECT=member-console-audit
: "${OUT:?set OUT to the directory that receives findings}"
mkdir -p "$OUT"; OUT="$(cd "$OUT" && pwd)"; export OUT
. ./harness.sh

# Reverify current IDs with `opencode models`; these drift. Gemini is in the
# default set
# (google/gemini-3.8-flash); its key is in the mounted auth.json and its host
# is allowlisted for egress. Override the set with MODELS="..." ./run-audit.sh.
DEFAULT_MODELS="deepseek/deepseek-v4-pro google/gemini-3.8-flash zai-coding-plan/glm-5.3 kimi-for-coding/k3-256k"
MODELS="${MODELS:-$DEFAULT_MODELS}"
# 1200s was too tight for Kimi at --variant max: it timed out mid-investigation
# on the larger slices (2026-09-08). Thorough models need real headroom.
TIMEOUT="${TIMEOUT:-2700}"
# Reasoning effort, in each harness's vocabulary (see harness.sh). "max" asks
# each model for its deepest reasoning; set VARIANT="" to omit the flag if a
# provider rejects the value.
VARIANT="${VARIANT:-max}"

if [[ ! -d .code ]]; then echo "Run ./prepare.sh first." >&2; exit 1; fi
harness_check_auth $MODELS
docker compose up -d egress-proxy >/dev/null

# The scanner pass is deterministic and cheap; run it once if it has not been.
if [[ ! -d "$OUT/tools" ]]; then
  echo "==> No scanner output yet; running ./run-tools.sh first"
  ./run-tools.sh
fi

# Which tasks: args override; else every scope file.
if [[ $# -gt 0 ]]; then
  TASKS=("$@")
else
  mapfile -t TASKS < <(cd scope && ls *.md | sed 's/\.md$//')
fi

RUN_ID="$(date +%Y-%m-%d_%H%M%S)"
RUN_DIR="$OUT/$RUN_ID"
mkdir -p "$RUN_DIR"
{
  echo "run:    $RUN_ID"
  echo "commit: $(cat .code/AUDIT_COMMIT.txt 2>/dev/null || echo '?')"
  echo "models: $MODELS"
  echo "variant: ${VARIANT:-<default>}"
  echo "tasks:  ${TASKS[*]}"
  echo "timeout-per-attempt: ${TIMEOUT}s"
  echo "quota-resume: every ${QUOTA_POLL}s, up to ${QUOTA_WAIT}s of waiting"
  echo "execution: ${SERIAL:+serial}${SERIAL:-parallel by model ($(echo $MODELS | wc -w) lanes)}"
} | tee "$RUN_DIR/manifest.txt"

# One lane per model: the models run concurrently, but within a lane the
# slices run serially, so each provider/key sees one request at a time (no
# rate-limit stacking). Set SERIAL=1 to fall back to a single sequential lane.
# Progress interleaves across lanes; each lane also logs to $outdir/_lane.log.
run_lane() {
  local model="$1"
  local harness id safe_model; harness="$(harness_of "$model")"; id="$(model_of "$model")"; safe_model="$(safe_name "$model")"
  local outdir="$RUN_DIR/$safe_model"; mkdir -p "$outdir"
  local lane_log="$outdir/_lane.log"
  # Lane state (a codex lane's throwaway CODEX_HOME) lives under .run/, which
  # git ignores and teardown.sh removes; harness_release removes it sooner.
  local state="$HERE/.run/$RUN_ID/$safe_model"; mkdir -p "$state"
  local -a LANE_DOCKER; mapfile -t LANE_DOCKER < <(harness_mount "$harness" "$state")
  local task file out err prompt env_sh plugins scan_dir
  for task in "${TASKS[@]}"; do
    task="${task%.md}"
    # A bare name is a slice under scope/; "open/<name>" is an open brief;
    # "codex-security/<name>" hands method and report to the Codex Security
    # plugin (codex harness only).
    [[ "$task" == */* ]] && file="$task.md" || file="scope/$task.md"
    if [[ ! -f "$file" ]]; then echo "  ! [$model] no $file, skipping"; continue; fi
    if [[ "$file" == codex-security/* && "$harness" != codex ]]; then
      echo "  ! [$model] $task needs the codex harness, skipping"; continue
    fi
    out="$outdir/${task//\//__}.md"; err="$outdir/${task//\//__}.err"
    echo "==> [$model] $task (started $(date +%H:%M:%S))"
    # The prompt. A slice: how to work, the system context as fact, the
    # slice, then what a finding must carry. An open brief: the brief, then
    # what a finding must carry; no map, no questions. A Codex Security task:
    # the task alone; the plugin owns the method and writes its own artifacts
    # (scan-manifest.json, findings.json, coverage.json, report.md) into a
    # scan directory under this lane's output, which the two variables name.
    env_sh=""; plugins=""
    case "$file" in
      scope/*) prompt="cat AUDITOR.md; echo; echo '---'; cat CONTEXT.md; echo; echo '---'; cat $file; echo; echo '---'; cat REPORT.md";;
      codex-security/*)
        scan_dir="$outdir/${task//\//__}"; mkdir -p "$scan_dir/state"
        prompt="cat $file; echo; echo 'Scan directory: /out/${scan_dir#"$OUT"/} (it exists; use that absolute path, and no other).'"
        env_sh="export CODEX_SECURITY_SCAN_ROOT='/out/${scan_dir#"$OUT"/}' CODEX_SECURITY_STATE_DIR='/out/${scan_dir#"$OUT"/}/state'"
        plugins="codex-security";;
      *) prompt="cat $file; echo; echo '---'; cat REPORT.md";;
    esac
    if ! harness_run_task "$harness" "$id" "$VARIANT" "$prompt" "$out" "$err" "$lane_log" "$env_sh" "$plugins"; then
      echo "   [$model] $task: exit non-zero or timeout — agent exit codes are unreliable; judging by output" >>"$lane_log"
    fi
    # opencode headless exits 0 even when it did nothing; trust the artifact.
    if [[ -s "$out" ]] && grep -qiE 'finding|COMPLETE|summary|report' "$out"; then
      echo "   ok   [$model] $task -> $out ($(wc -l <"$out") lines)"
    else
      echo "   SUSPECT [$model] $task -> EMPTY/SUSPECT; see $err"
    fi
  done
  harness_own_output "$outdir"
  harness_release "$harness" "$state"
  echo "== lane done: [$model]"
}

if [[ -n "${SERIAL:-}" ]]; then
  for model in $MODELS; do run_lane "$model"; done
else
  declare -a lane_pids=()
  for model in $MODELS; do
    run_lane "$model" &
    lane_pids+=("$!")
  done
  echo "==> ${#lane_pids[@]} model lanes running in parallel (pids: ${lane_pids[*]}); one provider/key per lane"
  wait "${lane_pids[@]}"
fi

echo
echo "Findings under: $RUN_DIR"
echo "Review, then verify each real finding against a fresh stack before it"
echo "enters the issue ledger in status/issues.md. These models over-report."