awatch-rus/AWatch-rus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(security): harden public secret scan and document policy

Browse Source
This commit is contained in:
igor04091968
2026-06-21 14:17:05 +03:00
parent 9f3278f0dc
commit 4f90aba2a1
15 changed files with 389 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/check_production_inventory_placeholders.sh
+1 -1
View File
@@ -86,7 +86,7 @@ if (( self_test == 1 )); then
cat >"$good" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://influxdb.internal:8086
AW_WORKTIME_INFLUX_HOSTS=WINDOWS-HOST
AW_WORKTIME_INFLUX_TOKEN=prod-write-token-value
AW_WORKTIME_INFLUX_TOKEN=dummy
EOF
cat >"$bad" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://192.0.2.10:8086
scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh
+1 -1
View File
@@ -86,7 +86,7 @@ if (( self_test == 1 )); then
cat >"$good" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://influxdb.internal:8086
AW_WORKTIME_INFLUX_HOSTS=WINDOWS-HOST
AW_WORKTIME_INFLUX_TOKEN=prod-write-token-value
AW_WORKTIME_INFLUX_TOKEN=dummy
EOF
cat >"$bad" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://192.0.2.10:8086
scripts/public_secret_pattern_check.py
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
"""Fail-closed public scan for obvious committed secrets.
The scanner intentionally prints only file, line and rule names. It never
prints the matched value.
"""
from __future__ import annotations
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
TEXT_SUFFIXES = {
".cfg",
".conf",
".env",
".ini",
".json",
".lock",
".md",
".py",
".rs",
".sh",
".toml",
".ts",
".txt",
".yaml",
".yml",
}
SKIP_DIRS = {
".git",
".mypy_cache",
".pytest_cache",
".ruff_cache",
"bin",
"dist",
"node_modules",
"release-evidence",
"target",
}
SKIP_FILES = {
"scripts/public_secret_pattern_check.py",
}
ALLOW_MARKERS = (
"# public-secret-scan: allow dummy",
"// public-secret-scan: allow dummy",
)
SAFE_LITERAL_VALUES = {
"",
"admin",
"change_me",
"change-me",
"changeme",
"dummy",
"example",
"placeholder",
"redacted",
"secret",
"test",
"<redacted>",
"<set_via_env>",
"<set-via-env>",
}
UNQUOTED_ASSIGNMENT_SUFFIXES = {
".cfg",
".conf",
".env",
".ini",
".sh",
".toml",
".yaml",
".yml",
}
SECRET_KEY_RE = re.compile(
r"(?i)\b(password|passwd|pwd|token|secret|api[_-]?key|bearer|cookie|private[_-]?key)\b"
)
TOKEN_LITERAL_RE = re.compile(r"^[A-Za-z0-9_./+=:-]{8,}$")
QUOTED_ASSIGNMENT_RE = re.compile(
r"(?ix)"
r"\b(?P<key>[A-Z0-9_./-]*(?:password|passwd|pwd|token|secret|api[_-]?key|bearer|cookie|private[_-]?key)[A-Z0-9_./-]*)\b"
r"\s*(?:[:=]|=>)\s*"
r"(?P<prefix>r|br|rb|R|BR|RB)?"
r"(?P<quote>['\"])(?P<value>[^'\"]{8,})(?P=quote)"
)
ENV_ASSIGNMENT_RE = re.compile(
r"(?i)^\s*(?:export\s+)?"
r"(?P<key>[A-Z0-9_./-]*(?:PASSWORD|PASSWD|PWD|TOKEN|SECRET|API[_-]?KEY|BEARER|COOKIE|PRIVATE[_-]?KEY)[A-Z0-9_./-]*)"
r"\s*=\s*(?P<value>[A-Za-z0-9_./+=:-]{8,})"
r"(?=\s*(?:#|$))"
)
PRIVATE_KEY_HEADER_RE = re.compile(
r"-----BEGIN (?:RSA |OPENSSH |EC |DSA )?PRIVATE KEY-----"
)
AWS_ACCESS_KEY_RE = re.compile(r"\bAKIA[0-9A-Z]{16}\b")
GITHUB_TOKEN_RE = re.compile(r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{30,}\b")
GITHUB_FINE_GRAINED_TOKEN_RE = re.compile(r"\bgithub_pat_[A-Za-z0-9_]{40,}\b")
SLACK_TOKEN_RE = re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b")
GOOGLE_API_KEY_RE = re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")
def is_allowlisted(line: str) -> bool:
return any(marker in line for marker in ALLOW_MARKERS)
def is_safe_literal(value: str) -> bool:
normalized = value.strip().strip("'\"").lower()
if normalized in SAFE_LITERAL_VALUES:
return True
if normalized.startswith(("{{", "{%")):
return True
if "{{" in normalized and "}}" in normalized:
return True
if normalized.startswith("<") and normalized.endswith(">"):
return True
if normalized.startswith(("env:", "env.", "process.env.", "${", "$")):
return True
if normalized.startswith(("c:\\", "/", "./", "../")):
return True
if "set_via_env" in normalized or "redacted" in normalized:
return True
return False
def quoted_assignment_findings(line: str) -> list[str]:
if "re.compile" in line:
return []
findings: list[str] = []
for match in QUOTED_ASSIGNMENT_RE.finditer(line):
value = match.group("value")
if not TOKEN_LITERAL_RE.match(value):
continue
if is_safe_literal(value):
continue
findings.append("secret_assignment")
return findings
def env_assignment_finding(line: str) -> str | None:
match = ENV_ASSIGNMENT_RE.search(line)
if not match:
return None
value = match.group("value")
if is_safe_literal(value):
return None
return "secret_assignment"
def scan_line(line: str, relative: Path) -> list[str]:
if is_allowlisted(line):
return []
findings: list[str] = []
if PRIVATE_KEY_HEADER_RE.search(line):
findings.append("private_key_header")
if AWS_ACCESS_KEY_RE.search(line):
findings.append("aws_access_key")
if GITHUB_TOKEN_RE.search(line) or GITHUB_FINE_GRAINED_TOKEN_RE.search(line):
findings.append("github_token")
if SLACK_TOKEN_RE.search(line):
findings.append("slack_token")
if GOOGLE_API_KEY_RE.search(line):
findings.append("google_api_key")
findings.extend(quoted_assignment_findings(line))
if relative.suffix.lower() in UNQUOTED_ASSIGNMENT_SUFFIXES:
env_finding = env_assignment_finding(line)
if env_finding:
findings.append(env_finding)
return sorted(set(findings))
def iter_text_files(root: Path):
candidates: list[Path]
try:
proc = subprocess.run(
["git", "-C", str(root), "ls-files", "-z"],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
candidates = [
root / item.decode("utf-8", errors="ignore")
for item in proc.stdout.split(b"\0")
if item
]
except (OSError, subprocess.CalledProcessError):
candidates = sorted(root.rglob("*"))
for path in candidates:
if not path.is_file():
continue
relative = path.relative_to(root)
relative_text = relative.as_posix()
if any(part in SKIP_DIRS for part in relative.parts):
continue
if relative_text in SKIP_FILES:
continue
if path.suffix.lower() not in TEXT_SUFFIXES:
continue
yield path, relative
def main() -> int:
findings: list[str] = []
for path, relative in iter_text_files(ROOT):
try:
lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
except OSError:
continue
for line_no, line in enumerate(lines, start=1):
for rule in scan_line(line, relative):
findings.append(f"{relative}:{line_no}:{rule}")
if findings:
print("secret_pattern_check=fail")
for finding in findings:
print(finding)
return 2
print("secret_pattern_check=ok")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/registry_readiness_check.sh
+17
View File
@@ -48,8 +48,10 @@ required_files=(
"docs/registry/registry-evidence-manifest.json"
"docs/PROJECT_STATUS_RU.md"
"docs/QUALITY_STATUS_RU.md"
"docs/SECURITY_SCANNING_POLICY_RU.md"
"scripts/build_release_evidence.sh"
"scripts/check_release_evidence.sh"
"scripts/public_secret_pattern_check.py"
".github/workflows/ci.yml"
".github/workflows/security.yml"
".github/workflows/coverage.yml"
@@ -201,6 +203,7 @@ require_grep "\"restore_tested\"[[:space:]]*:[[:space:]]*false" "docs/registry/r
require_grep "docs/registry" "docs/registry/WIKI_AND_DOCUMENTATION_POLICY_RU.md" "authoritative_docs_path_wiki_policy"
require_grep "release_evidence_check" "docs/registry/registry-evidence-manifest.json" "release_evidence_check_manifest"
require_grep "Public engineering transparency" "README.md" "readme_public_engineering_transparency"
require_grep "SECURITY_SCANNING_POLICY_RU\\.md" "README.md" "readme_security_scanning_policy"
require_grep "GitHub Actions is public mirror validation only|public mirror validation only" "docs/registry/RU_BUILD_RUNNER_READINESS_RU.md" "github_actions_not_registry_build_runner"
require_grep "GitHub Actions is public mirror validation only|public mirror validation only" "docs/registry/RELEASE_EVIDENCE_RUNBOOK_RU.md" "github_actions_not_registry_release_runbook"
require_grep "Public CI is not registry release evidence|not registry release evidence" "docs/QUALITY_STATUS_RU.md" "quality_public_ci_not_registry_evidence"
@@ -208,6 +211,9 @@ require_grep "requires_russian_build_runner|public_mirror_validation_only" "docs
require_grep "public mirror validation only" "SECURITY.md" "security_public_mirror_validation"
require_grep "public mirror validation only" "CONTRIBUTING.md" "contributing_public_mirror_validation"
require_grep "public mirror validation only" "ROADMAP.md" "roadmap_public_mirror_validation"
require_grep "fail-closed" "docs/SECURITY_SCANNING_POLICY_RU.md" "security_scanning_policy_fail_closed"
require_grep "public-secret-scan: allow dummy" "docs/SECURITY_SCANNING_POLICY_RU.md" "security_scanning_policy_allow_comment"
require_grep "scripts/public_secret_pattern_check\\.py" ".github/workflows/security.yml" "security_workflow_local_secret_scanner"
require_grep "cargo audit|cargo deny|secret-pattern" ".github/workflows/security.yml" "security_workflow_checks"
require_grep "cargo llvm-cov" ".github/workflows/coverage.yml" "coverage_workflow_llvm_cov"
require_grep "cargo fmt --all --check" ".github/workflows/ci.yml" "ci_workflow_fmt"
@@ -218,6 +224,7 @@ scan_files=(
"$REGISTRY_DIR"/*.md
"$REGISTRY_DIR"/*.json
"$ROOT/docs/QUALITY_STATUS_RU.md"
"$ROOT/docs/SECURITY_SCANNING_POLICY_RU.md"
"$ROOT/SECURITY.md"
"$ROOT/CONTRIBUTING.md"
"$ROOT/ROADMAP.md"
@@ -235,11 +242,21 @@ claim_scan_files=(
"$REGISTRY_DIR"/*.md
"$REGISTRY_DIR"/*.json
"$ROOT/docs/QUALITY_STATUS_RU.md"
"$ROOT/docs/SECURITY_SCANNING_POLICY_RU.md"
"$ROOT/SECURITY.md"
"$ROOT/CONTRIBUTING.md"
"$ROOT/ROADMAP.md"
)
if command -v python3 >/dev/null 2>&1; then
if ! python3 "$ROOT/scripts/public_secret_pattern_check.py" >/tmp/registry_public_secret_scan.$$ 2>&1; then
fail "public_secret_pattern_check:$(cat /tmp/registry_public_secret_scan.$$)"
fi
rm -f /tmp/registry_public_secret_scan.$$
else
printf 'warning: python3 not found; skipped public_secret_pattern_check\n' >&2
fi
if grep -RInEi "(password|passwd|pwd|token|secret|api[_-]?key|private[[:space:]_-]?key)[[:space:]]*[:=][[:space:]]*['\"]?[A-Za-z0-9_./+=-]{8,}" "${scan_files[@]}" >/tmp/registry_secret_like.$$ 2>/dev/null; then
fail "secret_like_value:$(cat /tmp/registry_secret_like.$$)"
fi