chore(security): harden public secret scan and document policy

This commit is contained in:
igor04091968
2026-06-21 14:17:05 +03:00
parent 9f3278f0dc
commit 4f90aba2a1
15 changed files with 389 additions and 61 deletions
+3 -48
View File
@@ -54,7 +54,8 @@ jobs:
uses: taiki-e/install-action@cargo-deny
- name: cargo deny
run: cargo deny --manifest-path adk-rust/Cargo.toml --config deny.toml check
run: cargo deny check advisories licenses sources --config ../deny.toml
working-directory: adk-rust
secret-pattern-check:
name: Secret pattern check
@@ -69,53 +70,7 @@ jobs:
lfs: false
- name: Scan for obvious secret patterns
run: |
python3 - <<'PY'
import os
import re
import sys
from pathlib import Path
root = Path(".")
text_suffixes = {
".cfg", ".conf", ".env", ".ini", ".json", ".lock", ".md", ".py",
".rs", ".sh", ".toml", ".ts", ".txt", ".yaml", ".yml"
}
skip_dirs = {
".git", "target", "node_modules", "release-evidence", "dist", "bin"
}
patterns = [
("private_key_header", re.compile(r"-----BEGIN (?:RSA |OPENSSH |EC |DSA )?PRIVATE KEY-----")),
("aws_access_key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
("secret_assignment", re.compile(
r"(?i)\b(password|passwd|pwd|token|secret|api[_-]?key|bearer|cookie)\b"
r"\s*[:=]\s*[\"']?[A-Za-z0-9_./+=-]{16,}"
)),
]
findings = []
for path in root.rglob("*"):
if not path.is_file():
continue
if any(part in skip_dirs for part in path.parts):
continue
if path.suffix.lower() not in text_suffixes:
continue
try:
lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
except OSError:
continue
for line_no, line in enumerate(lines, start=1):
for name, pattern in patterns:
if pattern.search(line):
findings.append(f"{path}:{line_no}:{name}")
break
if findings:
print("secret_pattern_check=fail")
for finding in findings:
print(finding)
sys.exit(2)
print("secret_pattern_check=ok")
PY
run: python3 scripts/public_secret_pattern_check.py
dependency-review:
name: Dependency review
+2
View File
@@ -242,6 +242,8 @@ collectors.
- Public CI, coverage baseline and security scanning are enabled on GitHub.
- Issue templates, PR template and public roadmap are maintained for process
visibility.
- Public secret scanning policy:
[docs/SECURITY_SCANNING_POLICY_RU.md](docs/SECURITY_SCANNING_POLICY_RU.md).
- GitHub remains public mirror validation only.
- Primary registry contour remains Gitea plus the Russian build-runner.
- Quality status:
+4 -4
View File
@@ -11609,7 +11609,7 @@ mod tests {
json_smoke: false,
evidence_only: false,
evidence_upload_token: None,
telemetry_api_key: "test-key".to_string(),
telemetry_api_key: "dummy".to_string(),
telemetry_store_path: dir.path().join("telemetry.jsonl"),
expected_nodes_path: dir.path().join("expected_nodes.json"),
security_events_backend: "disabled".to_string(),
@@ -11725,7 +11725,7 @@ mod tests {
json_smoke: false,
evidence_only: false,
evidence_upload_token: None,
telemetry_api_key: "test-key".to_string(),
telemetry_api_key: "dummy".to_string(),
telemetry_store_path: dir.path().join("telemetry.jsonl"),
expected_nodes_path: dir.path().join("expected_nodes.json"),
security_events_backend: "disabled".to_string(),
@@ -11804,7 +11804,7 @@ mod tests {
json_smoke: false,
evidence_only: false,
evidence_upload_token: None,
telemetry_api_key: "test-key".to_string(),
telemetry_api_key: "dummy".to_string(),
telemetry_store_path: dir.path().join("telemetry/telemetry.jsonl"),
expected_nodes_path: dir.path().join("expected_nodes.json"),
security_events_backend: "disabled".to_string(),
@@ -12474,7 +12474,7 @@ mod tests {
json_smoke: false,
evidence_only: false,
evidence_upload_token: None,
telemetry_api_key: "test-key".to_string(),
telemetry_api_key: "dummy".to_string(),
telemetry_store_path: case_dir.path().join("telemetry.jsonl"),
expected_nodes_path: case_dir.path().join("expected_nodes.json"),
security_events_backend: "disabled".to_string(),
@@ -239,7 +239,7 @@ mod tests {
json_smoke: false,
evidence_only: false,
evidence_upload_token: None,
telemetry_api_key: "test-key".to_string(),
telemetry_api_key: "dummy".to_string(),
telemetry_store_path: dir.join("telemetry.jsonl"),
expected_nodes_path: dir.join("expected_nodes.json"),
security_events_backend: "disabled".to_string(),
@@ -1045,7 +1045,7 @@ mod tests {
assert!(err.contains("AW_DLP_INFLUX_URL"));
config.influx_url = "http://influxdb.internal:8086".to_string();
config.influx_token = "prod-write-token-value".to_string();
config.influx_token = "dummy".to_string();
let err = validate_runtime_config(&config).unwrap_err().to_string();
assert!(err.contains("AW_DLP_INFLUX_HOSTS"));
@@ -1031,7 +1031,7 @@ mod tests {
assert!(err.contains("AW_WORKTIME_INFLUX_URL"));
config.influx_url = "http://influxdb.internal:8086".to_string();
config.influx_token = "prod-write-token-value".to_string();
config.influx_token = "dummy".to_string();
let err = validate_runtime_config(&config).unwrap_err().to_string();
assert!(err.contains("AW_WORKTIME_INFLUX_HOSTS"));
@@ -13,8 +13,8 @@ aw_pfsense_poller_config:
verify_tls: false
timeout_seconds: 15
auth:
api_key: "replace-me"
api_secret: "replace-me"
api_key: "<SET_VIA_ENV>"
api_secret: "<SET_VIA_ENV>"
endpoints:
- name: "system-status"
path: "/api/v2/status/system"
+1
View File
@@ -15,6 +15,7 @@ allow = [
"Apache-2.0 WITH LLVM-exception",
"BSD-2-Clause",
"BSD-3-Clause",
"CDLA-Permissive-2.0",
"CC0-1.0",
"ISC",
"MIT",
+2 -2
View File
@@ -97,7 +97,7 @@ git grep -n -E 'password|token|secret' -- \
ansible/group_vars/aw_server.yml:4:ansible_password: "{{ lookup('env', 'AW_SSH_PASSWORD') }}"
ansible/group_vars/aw_windows.yml:4:ansible_password: "{{ lookup('env', 'AW_WINRM_PASSWORD') }}"
ansible/group_vars/proxmox-bot.example.yml:1:telegram_bot_token: "CHANGE_ME"
ansible/group_vars/pfsense-poller.example.yml:17: api_secret: "replace-me"
ansible/group_vars/pfsense-poller.example.yml:17: api_secret: "<SET_VIA_ENV>"
docs/INSTALL_RU.md:66:- Telegram bot token;
docs/INSTALL_RU.md:67:- evidence upload token.
adk-rust/crates/detmir-portal/src/main.rs:1756:fn bearer_token(request: &Request) -> Option<String> {
@@ -167,7 +167,7 @@ logic и `CHANGE_ME`/`replace-me`. Реальные статические па
ansible/group_vars/aw_server.yml: ansible_password берется из AW_SSH_PASSWORD
ansible/group_vars/aw_windows.yml: ansible_password берется из AW_WINRM_PASSWORD
ansible/group_vars/proxmox-bot.example.yml: telegram_bot_token: "CHANGE_ME"
ansible/group_vars/pfsense-poller.example.yml: api_secret: "replace-me"
ansible/group_vars/pfsense-poller.example.yml: api_secret: "<SET_VIA_ENV>"
ansible/deploy_proxmox_web_gateway.yml: password генерируется через openssl rand
```
+76
View File
@@ -0,0 +1,76 @@
# Политика public secret scanning
Этот документ описывает публичную проверку репозитория на очевидные секреты.
Проверка нужна для инженерной прозрачности и снижения риска случайной
публикации токенов, паролей, cookies, private keys и похожих значений.
GitHub Actions используется как public mirror validation only. Основной
registry release evidence должен формироваться в российском build-контуре, а
не в GitHub Actions.
## Принцип проверки
- Scanner работает fail-closed: при подозрении на committed secret workflow
должен завершаться ошибкой.
- Scanner выводит только `file:line:rule` и не печатает найденное значение.
- Реальные секреты, токены, пароли, cookies, API keys и private keys нельзя
хранить в репозитории.
- Runtime-секреты должны передаваться через environment variables, защищенные
файлы вне репозитория или внешний secret storage.
- Документационные примеры должны использовать `<SET_VIA_ENV>`, `<REDACTED>`,
`example`, `dummy` или `redacted`.
## Тестовые значения
Для unit tests и fixtures допустимы только короткие безопасные значения:
- `dummy`
- `test`
- `example`
- `redacted`
- `secret`, если тест проверяет именно parsing поля и значение короткое
Не использовать длинные base64, hex, JWT-like или token-like строки даже в
тестах. Такие строки выглядят как настоящий secret и должны заменяться на
короткий dummy.
## Inline allow comments
Если строка безопасна, но scanner не может корректно определить контекст,
разрешен точечный inline allow comment:
```text
# public-secret-scan: allow dummy
```
```text
// public-secret-scan: allow dummy
```
Allow comment разрешен только для dummy/test fixtures, безопасных placeholder
values или runtime-derived значений, где секрет не хранится в репозитории.
Нельзя использовать allow comment для реального токена, пароля, cookie, private
key или customer evidence.
## Локальный запуск
```bash
python3 scripts/public_secret_pattern_check.py
```
Ожидаемый успешный результат:
```text
secret_pattern_check=ok
```
## Что делать при срабатывании
1. Проверить строку вручную.
2. Если значение настоящее, удалить его из истории рабочего изменения и
заменить на env/config reference.
3. Если значение тестовое, заменить на короткий dummy.
4. Если это безопасный placeholder или runtime-derived value, переписать строку
так, чтобы она не выглядела как секрет, либо добавить точечный inline allow
comment.
5. Повторить локальный запуск scanner и registry readiness check.
@@ -1,5 +1,39 @@
# Registry readiness changelog
## 2026-06-21 public secret scan hardening
Added:
- `scripts/public_secret_pattern_check.py` as a reproducible local equivalent
of the public GitHub Actions secret-pattern check.
- `docs/SECURITY_SCANNING_POLICY_RU.md` describing fail-closed public secret
scanning, dummy values and inline allow comments.
- README link to the public secret scanning policy.
- Registry readiness check integration for the local public secret scanner.
Changed:
- Security workflow now calls `python3 scripts/public_secret_pattern_check.py`
instead of inline Python.
- Secret scan output remains redacted and reports only `file:line:rule`.
- Cargo deny workflow command now runs from the Rust workspace and checks
advisories, licenses and sources with the repository `deny.toml`.
- `CDLA-Permissive-2.0` is explicitly allowed for `webpki-roots`; final
registry submission still requires legal review.
Runtime impact:
- No runtime/product code changes.
- No API changes.
- No UI changes.
- No deployment behavior changes.
Reason:
- First public security workflow exposed false positives on runtime-derived
values and safe config lookups. The scanner was hardened without disabling
the check and without broad directory allowlists.
## 2026-06-21 status freeze
Added:
@@ -86,7 +86,7 @@ if (( self_test == 1 )); then
cat >"$good" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://influxdb.internal:8086
AW_WORKTIME_INFLUX_HOSTS=WINDOWS-HOST
AW_WORKTIME_INFLUX_TOKEN=prod-write-token-value
AW_WORKTIME_INFLUX_TOKEN=dummy
EOF
cat >"$bad" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://192.0.2.10:8086
@@ -86,7 +86,7 @@ if (( self_test == 1 )); then
cat >"$good" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://influxdb.internal:8086
AW_WORKTIME_INFLUX_HOSTS=WINDOWS-HOST
AW_WORKTIME_INFLUX_TOKEN=prod-write-token-value
AW_WORKTIME_INFLUX_TOKEN=dummy
EOF
cat >"$bad" <<'EOF'
AW_WORKTIME_INFLUX_URL=http://192.0.2.10:8086
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
"""Fail-closed public scan for obvious committed secrets.
The scanner intentionally prints only file, line and rule names. It never
prints the matched value.
"""
from __future__ import annotations
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
TEXT_SUFFIXES = {
".cfg",
".conf",
".env",
".ini",
".json",
".lock",
".md",
".py",
".rs",
".sh",
".toml",
".ts",
".txt",
".yaml",
".yml",
}
SKIP_DIRS = {
".git",
".mypy_cache",
".pytest_cache",
".ruff_cache",
"bin",
"dist",
"node_modules",
"release-evidence",
"target",
}
SKIP_FILES = {
"scripts/public_secret_pattern_check.py",
}
ALLOW_MARKERS = (
"# public-secret-scan: allow dummy",
"// public-secret-scan: allow dummy",
)
SAFE_LITERAL_VALUES = {
"",
"admin",
"change_me",
"change-me",
"changeme",
"dummy",
"example",
"placeholder",
"redacted",
"secret",
"test",
"<redacted>",
"<set_via_env>",
"<set-via-env>",
}
UNQUOTED_ASSIGNMENT_SUFFIXES = {
".cfg",
".conf",
".env",
".ini",
".sh",
".toml",
".yaml",
".yml",
}
SECRET_KEY_RE = re.compile(
r"(?i)\b(password|passwd|pwd|token|secret|api[_-]?key|bearer|cookie|private[_-]?key)\b"
)
TOKEN_LITERAL_RE = re.compile(r"^[A-Za-z0-9_./+=:-]{8,}$")
QUOTED_ASSIGNMENT_RE = re.compile(
r"(?ix)"
r"\b(?P<key>[A-Z0-9_./-]*(?:password|passwd|pwd|token|secret|api[_-]?key|bearer|cookie|private[_-]?key)[A-Z0-9_./-]*)\b"
r"\s*(?:[:=]|=>)\s*"
r"(?P<prefix>r|br|rb|R|BR|RB)?"
r"(?P<quote>['\"])(?P<value>[^'\"]{8,})(?P=quote)"
)
ENV_ASSIGNMENT_RE = re.compile(
r"(?i)^\s*(?:export\s+)?"
r"(?P<key>[A-Z0-9_./-]*(?:PASSWORD|PASSWD|PWD|TOKEN|SECRET|API[_-]?KEY|BEARER|COOKIE|PRIVATE[_-]?KEY)[A-Z0-9_./-]*)"
r"\s*=\s*(?P<value>[A-Za-z0-9_./+=:-]{8,})"
r"(?=\s*(?:#|$))"
)
PRIVATE_KEY_HEADER_RE = re.compile(
r"-----BEGIN (?:RSA |OPENSSH |EC |DSA )?PRIVATE KEY-----"
)
AWS_ACCESS_KEY_RE = re.compile(r"\bAKIA[0-9A-Z]{16}\b")
GITHUB_TOKEN_RE = re.compile(r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{30,}\b")
GITHUB_FINE_GRAINED_TOKEN_RE = re.compile(r"\bgithub_pat_[A-Za-z0-9_]{40,}\b")
SLACK_TOKEN_RE = re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b")
GOOGLE_API_KEY_RE = re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")
def is_allowlisted(line: str) -> bool:
return any(marker in line for marker in ALLOW_MARKERS)
def is_safe_literal(value: str) -> bool:
normalized = value.strip().strip("'\"").lower()
if normalized in SAFE_LITERAL_VALUES:
return True
if normalized.startswith(("{{", "{%")):
return True
if "{{" in normalized and "}}" in normalized:
return True
if normalized.startswith("<") and normalized.endswith(">"):
return True
if normalized.startswith(("env:", "env.", "process.env.", "${", "$")):
return True
if normalized.startswith(("c:\\", "/", "./", "../")):
return True
if "set_via_env" in normalized or "redacted" in normalized:
return True
return False
def quoted_assignment_findings(line: str) -> list[str]:
if "re.compile" in line:
return []
findings: list[str] = []
for match in QUOTED_ASSIGNMENT_RE.finditer(line):
value = match.group("value")
if not TOKEN_LITERAL_RE.match(value):
continue
if is_safe_literal(value):
continue
findings.append("secret_assignment")
return findings
def env_assignment_finding(line: str) -> str | None:
match = ENV_ASSIGNMENT_RE.search(line)
if not match:
return None
value = match.group("value")
if is_safe_literal(value):
return None
return "secret_assignment"
def scan_line(line: str, relative: Path) -> list[str]:
if is_allowlisted(line):
return []
findings: list[str] = []
if PRIVATE_KEY_HEADER_RE.search(line):
findings.append("private_key_header")
if AWS_ACCESS_KEY_RE.search(line):
findings.append("aws_access_key")
if GITHUB_TOKEN_RE.search(line) or GITHUB_FINE_GRAINED_TOKEN_RE.search(line):
findings.append("github_token")
if SLACK_TOKEN_RE.search(line):
findings.append("slack_token")
if GOOGLE_API_KEY_RE.search(line):
findings.append("google_api_key")
findings.extend(quoted_assignment_findings(line))
if relative.suffix.lower() in UNQUOTED_ASSIGNMENT_SUFFIXES:
env_finding = env_assignment_finding(line)
if env_finding:
findings.append(env_finding)
return sorted(set(findings))
def iter_text_files(root: Path):
candidates: list[Path]
try:
proc = subprocess.run(
["git", "-C", str(root), "ls-files", "-z"],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
candidates = [
root / item.decode("utf-8", errors="ignore")
for item in proc.stdout.split(b"\0")
if item
]
except (OSError, subprocess.CalledProcessError):
candidates = sorted(root.rglob("*"))
for path in candidates:
if not path.is_file():
continue
relative = path.relative_to(root)
relative_text = relative.as_posix()
if any(part in SKIP_DIRS for part in relative.parts):
continue
if relative_text in SKIP_FILES:
continue
if path.suffix.lower() not in TEXT_SUFFIXES:
continue
yield path, relative
def main() -> int:
findings: list[str] = []
for path, relative in iter_text_files(ROOT):
try:
lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
except OSError:
continue
for line_no, line in enumerate(lines, start=1):
for rule in scan_line(line, relative):
findings.append(f"{relative}:{line_no}:{rule}")
if findings:
print("secret_pattern_check=fail")
for finding in findings:
print(finding)
return 2
print("secret_pattern_check=ok")
return 0
if __name__ == "__main__":
sys.exit(main())
+17
View File
@@ -48,8 +48,10 @@ required_files=(
"docs/registry/registry-evidence-manifest.json"
"docs/PROJECT_STATUS_RU.md"
"docs/QUALITY_STATUS_RU.md"
"docs/SECURITY_SCANNING_POLICY_RU.md"
"scripts/build_release_evidence.sh"
"scripts/check_release_evidence.sh"
"scripts/public_secret_pattern_check.py"
".github/workflows/ci.yml"
".github/workflows/security.yml"
".github/workflows/coverage.yml"
@@ -201,6 +203,7 @@ require_grep "\"restore_tested\"[[:space:]]*:[[:space:]]*false" "docs/registry/r
require_grep "docs/registry" "docs/registry/WIKI_AND_DOCUMENTATION_POLICY_RU.md" "authoritative_docs_path_wiki_policy"
require_grep "release_evidence_check" "docs/registry/registry-evidence-manifest.json" "release_evidence_check_manifest"
require_grep "Public engineering transparency" "README.md" "readme_public_engineering_transparency"
require_grep "SECURITY_SCANNING_POLICY_RU\\.md" "README.md" "readme_security_scanning_policy"
require_grep "GitHub Actions is public mirror validation only|public mirror validation only" "docs/registry/RU_BUILD_RUNNER_READINESS_RU.md" "github_actions_not_registry_build_runner"
require_grep "GitHub Actions is public mirror validation only|public mirror validation only" "docs/registry/RELEASE_EVIDENCE_RUNBOOK_RU.md" "github_actions_not_registry_release_runbook"
require_grep "Public CI is not registry release evidence|not registry release evidence" "docs/QUALITY_STATUS_RU.md" "quality_public_ci_not_registry_evidence"
@@ -208,6 +211,9 @@ require_grep "requires_russian_build_runner|public_mirror_validation_only" "docs
require_grep "public mirror validation only" "SECURITY.md" "security_public_mirror_validation"
require_grep "public mirror validation only" "CONTRIBUTING.md" "contributing_public_mirror_validation"
require_grep "public mirror validation only" "ROADMAP.md" "roadmap_public_mirror_validation"
require_grep "fail-closed" "docs/SECURITY_SCANNING_POLICY_RU.md" "security_scanning_policy_fail_closed"
require_grep "public-secret-scan: allow dummy" "docs/SECURITY_SCANNING_POLICY_RU.md" "security_scanning_policy_allow_comment"
require_grep "scripts/public_secret_pattern_check\\.py" ".github/workflows/security.yml" "security_workflow_local_secret_scanner"
require_grep "cargo audit|cargo deny|secret-pattern" ".github/workflows/security.yml" "security_workflow_checks"
require_grep "cargo llvm-cov" ".github/workflows/coverage.yml" "coverage_workflow_llvm_cov"
require_grep "cargo fmt --all --check" ".github/workflows/ci.yml" "ci_workflow_fmt"
@@ -218,6 +224,7 @@ scan_files=(
"$REGISTRY_DIR"/*.md
"$REGISTRY_DIR"/*.json
"$ROOT/docs/QUALITY_STATUS_RU.md"
"$ROOT/docs/SECURITY_SCANNING_POLICY_RU.md"
"$ROOT/SECURITY.md"
"$ROOT/CONTRIBUTING.md"
"$ROOT/ROADMAP.md"
@@ -235,11 +242,21 @@ claim_scan_files=(
"$REGISTRY_DIR"/*.md
"$REGISTRY_DIR"/*.json
"$ROOT/docs/QUALITY_STATUS_RU.md"
"$ROOT/docs/SECURITY_SCANNING_POLICY_RU.md"
"$ROOT/SECURITY.md"
"$ROOT/CONTRIBUTING.md"
"$ROOT/ROADMAP.md"
)
if command -v python3 >/dev/null 2>&1; then
if ! python3 "$ROOT/scripts/public_secret_pattern_check.py" >/tmp/registry_public_secret_scan.$$ 2>&1; then
fail "public_secret_pattern_check:$(cat /tmp/registry_public_secret_scan.$$)"
fi
rm -f /tmp/registry_public_secret_scan.$$
else
printf 'warning: python3 not found; skipped public_secret_pattern_check\n' >&2
fi
if grep -RInEi "(password|passwd|pwd|token|secret|api[_-]?key|private[[:space:]_-]?key)[[:space:]]*[:=][[:space:]]*['\"]?[A-Za-z0-9_./+=-]{8,}" "${scan_files[@]}" >/tmp/registry_secret_like.$$ 2>/dev/null; then
fail "secret_like_value:$(cat /tmp/registry_secret_like.$$)"
fi