cgalo5758
5829091881
The contained runner that drove the 2026-09 security audit, the README review and four rounds of design ideation lived only inside the ignored notebook. It moves to scripts/agent-runner/: the prepare, tools, audit, ideation and teardown scripts, the compose and container files, the allowlist proxy and the prompt templates, with the paths that assumed the notebook fixed and findings written to a caller-named directory. Run outputs, transcripts and the round-specific sheet scripts stay behind. docs/agent-runner.md states the method: the disposable git archive copy and the fail-closed proxy, how a task is shaped, union rather than intersection of findings across models, separate adjudication of every finding against the source, and the evidence a finding must carry.
114 lines
4.8 KiB
Bash
Executable File
114 lines
4.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Ralph-style audit loop: for each model, for each scope task, run a FRESH
|
|
# auditor container (clean context) and capture its findings. Sequential by
|
|
# design — concurrent opencode runs in one project deadlock, and it respects
|
|
# provider rate limits.
|
|
#
|
|
# Usage:
|
|
# OUT=<dir> ./run-audit.sh # default models, all tasks
|
|
# OUT=<dir> MODELS="deepseek/deepseek-v4-pro" ./run-audit.sh
|
|
# OUT=<dir> ./run-audit.sh 02-authorization 05-csp-headers # only these tasks
|
|
# OUT=<dir> TIMEOUT=1800 ./run-audit.sh
|
|
# OUT=<dir> VARIANT=high ./run-audit.sh # reasoning effort; default max, "" to omit
|
|
#
|
|
# OUT names the directory that receives findings. Keep it outside this one:
|
|
# this directory is tracked, and a run's output is not.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
PROJECT=member-console-audit
|
|
: "${OUT:?set OUT to the directory that receives findings}"
|
|
mkdir -p "$OUT"; OUT="$(cd "$OUT" && pwd)"; export OUT
|
|
|
|
# Reverify current IDs with `opencode models`; these drift. Gemini is in the
|
|
# default set
|
|
# (google/gemini-3.8-flash); its key is in the mounted auth.json and its host
|
|
# is allowlisted for egress. Override the set with MODELS="..." ./run-audit.sh.
|
|
DEFAULT_MODELS="deepseek/deepseek-v4-pro google/gemini-3.8-flash zai-coding-plan/glm-5.3 kimi-for-coding/k3-256k"
|
|
MODELS="${MODELS:-$DEFAULT_MODELS}"
|
|
# 1200s was too tight for Kimi at --variant max: it timed out mid-investigation
|
|
# on the larger slices (2026-09-08). Thorough models need real headroom.
|
|
TIMEOUT="${TIMEOUT:-2700}"
|
|
# Provider-specific reasoning effort. "max" asks each model for its deepest
|
|
# reasoning; set VARIANT="" to omit the flag if a provider rejects the value.
|
|
VARIANT="${VARIANT:-max}"
|
|
VARIANT_FLAG=""
|
|
[[ -n "$VARIANT" ]] && VARIANT_FLAG="--variant '$VARIANT'"
|
|
|
|
if [[ ! -d .code ]]; then echo "Run ./prepare.sh first." >&2; exit 1; fi
|
|
docker compose up -d egress-proxy >/dev/null
|
|
|
|
# The scanner pass is deterministic and cheap; run it once if it has not been.
|
|
if [[ ! -d "$OUT/tools" ]]; then
|
|
echo "==> No scanner output yet; running ./run-tools.sh first"
|
|
./run-tools.sh
|
|
fi
|
|
|
|
# Which tasks: args override; else every scope file.
|
|
if [[ $# -gt 0 ]]; then
|
|
TASKS=("$@")
|
|
else
|
|
mapfile -t TASKS < <(cd scope && ls *.md | sed 's/\.md$//')
|
|
fi
|
|
|
|
RUN_ID="$(date +%Y-%m-%d_%H%M%S)"
|
|
RUN_DIR="$OUT/$RUN_ID"
|
|
mkdir -p "$RUN_DIR"
|
|
{
|
|
echo "run: $RUN_ID"
|
|
echo "commit: $(cat .code/AUDIT_COMMIT.txt 2>/dev/null || echo '?')"
|
|
echo "models: $MODELS"
|
|
echo "variant: ${VARIANT:-<default>}"
|
|
echo "tasks: ${TASKS[*]}"
|
|
echo "timeout-per-iteration: ${TIMEOUT}s"
|
|
echo "execution: ${SERIAL:+serial}${SERIAL:-parallel by model ($(echo $MODELS | wc -w) lanes)}"
|
|
} | tee "$RUN_DIR/manifest.txt"
|
|
|
|
# One lane per model: the models run concurrently, but within a lane the
|
|
# slices run serially, so each provider/key sees one request at a time (no
|
|
# rate-limit stacking). Set SERIAL=1 to fall back to a single sequential lane.
|
|
# Progress interleaves across lanes; each lane also logs to $outdir/_lane.log.
|
|
run_lane() {
|
|
local model="$1"
|
|
local safe_model="${model//\//__}"
|
|
local outdir="$RUN_DIR/$safe_model"; mkdir -p "$outdir"
|
|
local lane_log="$outdir/_lane.log"
|
|
local task out err
|
|
for task in "${TASKS[@]}"; do
|
|
task="${task%.md}"
|
|
if [[ ! -f "scope/$task.md" ]]; then echo " ! [$model] no scope/$task.md, skipping"; continue; fi
|
|
out="$outdir/$task.md"; err="$outdir/$task.err"
|
|
echo "==> [$model] $task (started $(date +%H:%M:%S))"
|
|
# Fresh --rm container. --no-deps: don't restart the proxy. -T: no TTY.
|
|
# Prompt = auditor instructions + this task. stdin closed (headless quirk).
|
|
if timeout "$TIMEOUT" docker compose run --rm --no-deps -T auditor \
|
|
bash -c "opencode run --model '$model' $VARIANT_FLAG \"\$(cat AUDITOR.md; echo; echo '---'; cat CONTEXT.md; echo; echo '---'; cat scope/$task.md)\" </dev/null" \
|
|
>"$out" 2>"$err"; then :; else
|
|
echo " [$model] $task: exit non-zero or timeout — opencode exit codes are unreliable; judging by output" >>"$lane_log"
|
|
fi
|
|
# opencode headless exits 0 even when it did nothing; trust the artifact.
|
|
if [[ -s "$out" ]] && grep -qiE 'finding|COMPLETE|summary' "$out"; then
|
|
echo " ok [$model] $task -> $out ($(wc -l <"$out") lines)"
|
|
else
|
|
echo " SUSPECT [$model] $task -> EMPTY/SUSPECT; see $err"
|
|
fi
|
|
done
|
|
echo "== lane done: [$model]"
|
|
}
|
|
|
|
if [[ -n "${SERIAL:-}" ]]; then
|
|
for model in $MODELS; do run_lane "$model"; done
|
|
else
|
|
declare -a lane_pids=()
|
|
for model in $MODELS; do
|
|
run_lane "$model" &
|
|
lane_pids+=("$!")
|
|
done
|
|
echo "==> ${#lane_pids[@]} model lanes running in parallel (pids: ${lane_pids[*]}); one provider/key per lane"
|
|
wait "${lane_pids[@]}"
|
|
fi
|
|
|
|
echo
|
|
echo "Findings under: $RUN_DIR"
|
|
echo "Review, then verify each real finding against a fresh stack before it"
|
|
echo "enters the issue ledger in status/issues.md. These models over-report."
|