#!/usr/bin/env python3 """Revivable pre-publish scanner: finds secrets and private data before a project is shared. Usage: python3 -I scan.py [--history] [--json out.json] --history also scan every line ever added in the git history Stdlib only. Never prints a full secret: values are masked (first 4 chars + length). Exit code: 0 = nothing found (OK), 1 = only medium/low findings (WARN candidate), 3 = critical/high findings (BLOCK candidate), 2 = usage error. The verdict is only a candidate: an agent or human must judge each finding (real / test data / false positive). """ import json import math import os import re import subprocess import sys SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", "bin", "obj", ".pio", "Library", "Temp", "Logs", ".gradle", ".idea", ".next", "target", "vendor"} MAX_FILE = 2_000_000 BINARY_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".tar", ".7z", ".mp3", ".mp4", ".wav", ".ogg", ".fbx", ".blend", ".glb", ".gltf", ".obj", ".psd", ".ttf", ".otf", ".woff", ".woff2", ".dll", ".exe", ".so", ".dylib", ".class", ".jar", ".pptx", ".docx", ".xlsx", ".asset", ".unity", ".prefab", ".mat", ".meta", ".bin", ".elf", ".apk", ".aab"} # (id, severity, description, regex) severity: critical | high | medium | low RULES = [ ("private-key", "critical", "Private key (PEM/SSH)", r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY"), ("aws-access-key", "critical", "AWS access key", r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), ("openai-key", "critical", "OpenAI API key", r"\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_-]{20,}\b"), ("anthropic-key", "critical", "Anthropic API key", r"\bsk-ant-[A-Za-z0-9_-]{20,}\b"), ("github-token", "critical", "GitHub token", r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,})\b"), ("gitlab-token", "critical", "GitLab token", r"\bglpat-[A-Za-z0-9_-]{20,}\b"), ("slack-token", "critical", "Slack token", r"\bxox[abposr]-[A-Za-z0-9-]{10,}\b"), ("stripe-key", "critical", "Stripe secret key", r"\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b"), ("google-api-key", "critical", "Google API key", r"\bAIza[0-9A-Za-z_-]{35}\b"), ("telegram-bot-token", "critical", "Telegram bot token", r"\b\d{8,10}:AA[A-Za-z0-9_-]{30,}\b"), ("discord-token", "critical", "Discord bot token", r"\b[MNO][A-Za-z\d_-]{23,25}\.[A-Za-z\d_-]{6}\.[A-Za-z\d_-]{27,}\b"), ("rapidapi-key", "critical", "RapidAPI key", r"\b[0-9a-zA-Z]{8,}msh[0-9a-zA-Z]{8,}jsn[0-9a-zA-Z]{6,}\b"), ("key-in-header-or-call", "high", "Key/token passed as literal (header, call argument)", r"(?i)[\"'](?:x-[a-z-]*(?:key|token|secret)|[a-z_-]*(?:api|secret|private|access)[_-]?(?:key|token))[\"']\s*[,:=]\s*" r"[\"']([A-Za-z0-9_\-.]{16,})[\"']"), ("property-secret", "high", "Secret in a properties/ini/yaml setting", r"(?im)^\s*[\w.\-]*(?:password|passwd|secret|token|api[_-]?key)\s*[=:]\s*[\"']?([^\s#;\"']{3,})"), ("jwt", "high", "JSON Web Token", r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"), ("url-credentials", "critical", "Credentials inside a URL / connection string", r"\b[a-z][a-z0-9+.-]{1,20}://[^\s:/@'\"]{1,64}:[^\s@'\"]{3,128}@[^\s'\"]+"), ("password-assignment", "high", "Password/secret assigned in code or config", r"(?i)\b(?:pass(?:word|wd)?|pwd|secret|api[_-]?key|apikey|access[_-]?token|auth[_-]?token|token|client[_-]?secret)" r"\b[\"']?\s*[:=]\s*[\"']([^\"'\s,;}{<>]{6,})[\"']?"), ("db-password-param", "high", "Password in a connection string", r"(?i)\b(?:password|pwd)\s*=\s*([^;\"'\s]{3,})"), ("iban", "high", "IBAN (bank account)", r"\b[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){3,7}(?: ?[A-Z0-9]{1,4})?\b"), ("email", "medium", "E-mail address", r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), ("phone", "medium", "Phone number", r"(?]*>|\$\{[^}]*\}|\{\{[^}]*\}\}|%\w+%|process\.env.*|os\.environ.*|" r"null|none|true|false|todo|xxx.*)$") SAFE_EMAIL = re.compile(r"(?i)@(example\.(com|org|net)|test\.|localhost|users\.noreply\.github\.com|anthropic\.com)") SAFE_IP = {"0.0.0.0", "127.0.0.1", "255.255.255.255", "1.1.1.1", "8.8.8.8", "8.8.4.4"} VERSION_LIKE = re.compile(r"^\d+\.\d+\.\d+\.\d+$") def entropy(s): if not s: return 0.0 freq = {c: s.count(c) for c in set(s)} return -sum(n / len(s) * math.log2(n / len(s)) for n in freq.values()) def mask(v): # type and length only - no characters of the value, so output can be pasted safely return f"<{len(v.strip())} chars>" def is_text(path): if os.path.splitext(path)[1].lower() in BINARY_EXT: return False try: with open(path, "rb") as f: return b"\0" not in f.read(4096) except OSError: return False KNOWN_EXAMPLES = {"DE89370400440532013000"} # the textbook example IBAN GENERATED_DIRS = {".vscode", ".vs", "UserSettings", ".idea", ".settings"} def gitignored(root, paths): """Return the subset of paths that git would ignore (empty if no git repo).""" if not paths or not os.path.exists(os.path.join(root, ".git")): return set() try: r = subprocess.run(["git", "-C", root, "check-ignore", "--stdin"], input="\n".join(sorted(paths)), capture_output=True, text=True, timeout=60) return set(r.stdout.split("\n")) - {""} except (OSError, subprocess.TimeoutExpired): return set() def check_line(line, rel, where, findings): if len(line) > 5000: line = line[:5000] for rid, sev, desc, rx in COMPILED: for m in rx.finditer(line): value = m.group(1) if m.groups() and m.group(1) else m.group(0) if rid in ("password-assignment", "db-password-param", "property-secret", "key-in-header-or-call"): if PLACEHOLDER.match(value) or entropy(value) < 2.0: continue if rid == "email" and (SAFE_EMAIL.search(value) or re.search(r"://[^\s]*" + re.escape(value), line)): continue if rid in ("public-ip", "private-ip") and (value in SAFE_IP or rel.endswith(("package-lock.json", ".lock"))): continue if rid == "public-ip" and (VERSION_LIKE.match(value) and re.search(r"(?i)version|v\d", line)): continue if rid == "iban": compact = value.replace(" ", "") if not re.search(r"\d{8,}", compact): continue if compact in KNOWN_EXAMPLES: sev, desc = "low", "IBAN (known example number)" findings.append({"rule": rid, "severity": sev, "what": desc, "file": rel, "line": where, "preview": mask(value)}) def scan_history(root): """Scan lines added in any commit of the git history (secrets survive deletion there).""" findings = [] try: log = subprocess.run(["git", "-C", root, "log", "-p", "--all", "--no-color", "--unified=0", "--format=commit %h %ae"], capture_output=True, text=True, errors="replace", timeout=300).stdout except (OSError, subprocess.TimeoutExpired): return findings commit, rel, authors = "?", "?", set() for line in log.splitlines(): if line.startswith("commit "): parts = line.split() commit = parts[1] if len(parts) > 2 and not SAFE_EMAIL.search(parts[2]) and parts[2] not in authors: authors.add(parts[2]) findings.append({"rule": "author-email", "severity": "medium", "what": "Commit author e-mail (published with the history)", "file": "(git history)", "line": f"history@{commit}", "preview": mask(parts[2])}) elif line.startswith("+++ "): rel = line[6:] if line.startswith("+++ b/") else line[4:] elif line.startswith("+") and not line.startswith("+++"): check_line(line[1:], rel, f"history@{commit}", findings) for f in findings: if f["rule"] != "author-email": f["rule"] = "history:" + f["rule"] return findings def scan(root): findings, files_scanned, skipped = [], 0, 0 for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] for name in filenames: path = os.path.join(dirpath, name) rel = os.path.relpath(path, root) if rel == ".git": continue # worktree pointer file, never published if os.path.islink(path): continue # symlink: the target is scanned on its own for rx, sev, desc in SENSITIVE_FILES: if rx.search(rel) and not rel.endswith((".example", ".sample", ".template")): findings.append({"rule": "sensitive-file", "severity": sev, "what": desc, "file": rel, "line": None, "preview": None}) try: if os.path.getsize(path) > MAX_FILE or not is_text(path): skipped += 1 continue with open(path, encoding="utf-8", errors="replace") as f: lines = f.readlines() except OSError: continue files_scanned += 1 for no, line in enumerate(lines, 1): check_line(line, rel, no, findings) # de-duplicate (same rule, file, line) seen, unique = set(), [] for f in findings: key = (f["rule"], f["file"], f["line"]) if key not in seen: seen.add(key) unique.append(f) return unique, files_scanned, skipped def main(): args = sys.argv[1:] if not args or args[0] in ("-h", "--help"): print(__doc__) return 2 root = args[0] out = args[args.index("--json") + 1] if "--json" in args else None findings, n, n_skipped = scan(root) has_git = os.path.exists(os.path.join(root, ".git")) # dir, or file in worktrees/submodules if "--history" in args and has_git: findings += scan_history(root) order = {"critical": 0, "high": 1, "medium": 2, "low": 3} findings.sort(key=lambda f: (order[f["severity"]], f["file"], (0, f["line"]) if isinstance(f["line"], int) else (1, str(f["line"] or "")))) ignored = gitignored(root, {f["file"] for f in findings if not f["rule"].startswith("history:")}) for f in findings: if f["file"] in ignored and not f["rule"].startswith("history:"): f["ignored_by_git"] = True f["what"] += " [gitignored]" if f["file"].split("/")[0] in GENERATED_DIRS: f["generated"] = True f["what"] += " [IDE/generated]" summary = {s: sum(1 for f in findings if f["severity"] == s) for s in order} if out: with open(out, "w") as fh: json.dump({"root": os.path.abspath(root), "files_scanned": n, "summary": summary, "findings": findings}, fh, indent=2) print(f"Scanned {n} text files in {root}; skipped {n_skipped} binary/large files " f"(images, models, archives: check origin and content by hand) and these dirs: {', '.join(sorted(SKIP_DIRS))}") print("Summary: " + ", ".join(f"{k}={v}" for k, v in summary.items())) # group: more than 3 hits of the same rule in one file -> one line groups = {} for f in findings: groups.setdefault((f["severity"], f["rule"], f["what"], f["file"]), []).append(f) shown = 0 for (sev, _, what, file), items in groups.items(): if shown >= 150: print("... more (see --json)") break shown += 1 if len(items) > 3: print(f"[{sev.upper():8}] {what:45} {file} {len(items)}x (lines {items[0]['line']}…{items[-1]['line']})") continue for f in items: loc = f"{f['file']}:{f['line']}" if f["line"] else f["file"] print(f"[{sev.upper():8}] {what:45} {loc} {f['preview'] or ''}") if has_git and "--history" not in args: print("NOTE: git history not scanned - rerun with --history (secrets in old commits survive deletion).") if "--history" in args and not has_git: print("NOTE: no git history found in this folder - nothing to scan there.") if any(f.get("ignored_by_git") for f in findings): print("NOTE: [gitignored] files are not in the repo, but WOULD be included in a ZIP upload.") if summary["critical"] or summary["high"]: print("VERDICT: BLOCK candidate - review every critical/high finding before publishing.") return 3 if findings: print("VERDICT: WARN candidate - only medium/low findings.") return 1 print("VERDICT: OK - nothing found by patterns (still check images/binaries by hand).") return 0 if __name__ == "__main__": sys.exit(main())