Kai
88b06d782f
The curator Python backend (package, tests, systemd units, config templates, scripts) now lives under scenarios/curator/backend and is the single source of truth; the live checkout at the workspace path is a runtime copy. Exported from the app repo's tracked tree via git archive (no history, .pi/venv/caches excluded). 149 unit tests pass from the new location. profile.toml backend is now repo-relative (scenarios/curator/backend); verify-generated.sh resolves a relative backend against REPO_ROOT. verify-no-secrets ASSIGN heuristic now requires value entropy so vendored kwargs like token=extraction_token no longer false-positive. README documents the backend/ layout and the operator-owned app rollout step.
132 lines
4.9 KiB
Bash
Executable File
132 lines
4.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ---------------------------------------------------------------------------
|
|
# Refuse to commit credentials.
|
|
#
|
|
# This repository stores Pi configuration for several agent scenarios. Some of
|
|
# the files it mirrors (models.json, auth.json, *.env) legitimately contain
|
|
# plaintext API keys on the live host. Templates with ${PLACEHOLDER} markers
|
|
# belong in git; rendered files with real values never do.
|
|
#
|
|
# Usage:
|
|
# scripts/verify-no-secrets.sh # scan staged changes (pre-commit)
|
|
# scripts/verify-no-secrets.sh --all # scan every tracked file
|
|
#
|
|
# Install as a hook:
|
|
# ln -sf ../../scripts/verify-no-secrets.sh .git/hooks/pre-commit
|
|
# ---------------------------------------------------------------------------
|
|
set -uo pipefail
|
|
|
|
# Resolve the repository root via git, not via BASH_SOURCE: when this script is
|
|
# invoked through the .git/hooks/pre-commit symlink, dirname(BASH_SOURCE)/..
|
|
# resolves to .git/ rather than the work tree.
|
|
REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || {
|
|
echo "verify-no-secrets: not inside a git work tree" >&2
|
|
exit 1
|
|
}
|
|
cd "$REPO_ROOT" || exit 1
|
|
|
|
MODE="${1:-staged}"
|
|
|
|
if [ "$MODE" = "--all" ]; then
|
|
mapfile -t FILES < <(git ls-files)
|
|
else
|
|
mapfile -t FILES < <(git diff --cached --name-only --diff-filter=ACMR)
|
|
fi
|
|
|
|
if [ "${#FILES[@]}" -eq 0 ]; then
|
|
echo "verify-no-secrets: nothing to scan"
|
|
exit 0
|
|
fi
|
|
|
|
FAILED=0
|
|
|
|
fail() {
|
|
printf '\033[31mBLOCKED\033[0m %s\n %s\n' "$1" "$2" >&2
|
|
FAILED=1
|
|
}
|
|
|
|
# --- Rule 1: filenames that must never be committed ------------------------
|
|
FORBIDDEN_NAMES='(^|/)(models\.json|auth\.json|trust\.json|models-store\.json)$|(^|/)\.env$|\.env\.[^/]*$|(^|/)id_(ed25519|rsa)|\.(pem|p12)$|\.rendered(\..*)?$'
|
|
# Inside secrets/ only documentation and placeholder scaffolding may be tracked.
|
|
SECRETS_ALLOWED='(^|/)secrets/(\.gitkeep|README\.md|.*\.(example|template))$'
|
|
for f in "${FILES[@]}"; do
|
|
# Templates and examples are the intended way to track credential-shaped files.
|
|
case "$f" in
|
|
*.env.example|*.env.template|*.example|*.template) continue ;;
|
|
esac
|
|
if printf '%s' "$f" | grep -qP "$FORBIDDEN_NAMES"; then
|
|
fail "$f" "filename is on the never-commit list"
|
|
continue
|
|
fi
|
|
if printf '%s' "$f" | grep -qP '(^|/)secrets/' \
|
|
&& ! printf '%s' "$f" | grep -qP "$SECRETS_ALLOWED"; then
|
|
fail "$f" "files under secrets/ may only be .gitkeep, README.md, *.example or *.template"
|
|
fi
|
|
done
|
|
|
|
# --- Rule 2: high-confidence credential content ----------------------------
|
|
# Only applied to text files. Placeholders are explicitly tolerated.
|
|
PLACEHOLDER='\$\{[A-Z_][A-Z0-9_]*\}|<[A-Z_][A-Z0-9_]*>|REDACTED|CHANGE_?ME|EXAMPLE|xxx+|\.\.\.'
|
|
|
|
declare -a PATTERNS=(
|
|
'sk-[A-Za-z0-9]{20,}' # OpenAI-style
|
|
'sk-ant-[A-Za-z0-9_-]{20,}' # Anthropic
|
|
'ghp_[A-Za-z0-9]{30,}' # GitHub PAT
|
|
'gho_[A-Za-z0-9]{30,}'
|
|
'github_pat_[A-Za-z0-9_]{30,}'
|
|
'AIza[A-Za-z0-9_-]{30,}' # Google
|
|
'xox[baprs]-[A-Za-z0-9-]{10,}' # Slack
|
|
'[0-9]{8,10}:AA[A-Za-z0-9_-]{30,}' # Telegram bot token
|
|
'tvly-[A-Za-z0-9]{20,}' # Tavily
|
|
'-----BEGIN [A-Z ]*PRIVATE KEY-----'
|
|
)
|
|
|
|
# key-ish assignment with a long opaque value. The value must carry entropy (a
|
|
# digit or an uppercase letter): real secrets are base64/hex/random, while
|
|
# snake_case source identifiers like `token=extraction_token` are not, and used
|
|
# to trip this rule once the Python backend was vendored into the repo.
|
|
ASSIGN='(?i)(api[_-]?key|apikey|secret|token|password|passwd|access[_-]?key)["'"'"' ]*[:=]["'"'"' ]*(?-i:(?=[A-Za-z0-9/_+=-]{16,})(?=[A-Za-z0-9/_+=-]*[A-Z0-9]))[A-Za-z0-9/_+=-]{16,}'
|
|
|
|
for f in "${FILES[@]}"; do
|
|
[ -f "$f" ] || continue
|
|
# skip binaries
|
|
if ! grep -Iq . "$f" 2>/dev/null; then continue; fi
|
|
|
|
for pat in "${PATTERNS[@]}"; do
|
|
if hit=$(grep -nP "$pat" "$f" 2>/dev/null | head -1); then
|
|
[ -n "$hit" ] && fail "$f" "credential pattern: ${hit:0:120}"
|
|
fi
|
|
done
|
|
|
|
if hit=$(grep -nP "$ASSIGN" "$f" 2>/dev/null | grep -vP "$PLACEHOLDER" | head -1); then
|
|
[ -n "$hit" ] && fail "$f" "credential-like assignment: ${hit:0:120}"
|
|
fi
|
|
done
|
|
|
|
if [ "$FAILED" -ne 0 ]; then
|
|
cat >&2 <<'MSG'
|
|
|
|
verify-no-secrets: commit refused.
|
|
|
|
If a match is a false positive, either
|
|
- replace the value with a ${PLACEHOLDER} and keep the file as a *.template, or
|
|
- move the file outside the repository, or
|
|
- add a narrow exception to scripts/verify-no-secrets.sh with a comment
|
|
explaining why the value is not a credential.
|
|
|
|
Do not bypass with --no-verify.
|
|
MSG
|
|
exit 1
|
|
fi
|
|
|
|
echo "verify-no-secrets: ${#FILES[@]} file(s) scanned, clean"
|
|
|
|
# Generated regions must match the code that generates them. A stale tool list in
|
|
# a system prompt tells the model about tools that do not exist, or hides ones
|
|
# that do, and the symptom is an agent answering from memory.
|
|
REPO="$(git rev-parse --show-toplevel)"
|
|
if [ -x "$REPO/scripts/verify-generated.sh" ]; then
|
|
"$REPO/scripts/verify-generated.sh" || exit 1
|
|
fi
|
|
exit 0
|