diff --git a/DEVELOPMENT_PLAN_NEXT.md b/DEVELOPMENT_PLAN_NEXT.md index af008bd..1c2a36e 100644 --- a/DEVELOPMENT_PLAN_NEXT.md +++ b/DEVELOPMENT_PLAN_NEXT.md @@ -230,6 +230,10 @@ Expected benefit: long-running production operation has bounded disk behavior. ### P0-5. Remove ClickHouse password exposure from process arguments +Status: addressed by TASK_068. Runtime ClickHouse/1C wrappers keep +`CLICKHOUSE_PASSWORD` in the environment/config path and no longer pass it in +process arguments. + Purpose: keep production credentials out of `ps`/process argv. Reason: `rg` confirmed `clickhouse-1c/ops/run_*.sh` wrappers pass diff --git a/clickhouse-1c/ops/check_ingest_freshness.sh b/clickhouse-1c/ops/check_ingest_freshness.sh index b8aebeb..2037240 100644 --- a/clickhouse-1c/ops/check_ingest_freshness.sh +++ b/clickhouse-1c/ops/check_ingest_freshness.sh @@ -18,13 +18,12 @@ fi # shellcheck disable=SC1090 . "${ENV_FILE}" +# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh +. "${ROOT}/ops/clickhouse-client-safe.sh" query_max_age() { local table="$1" - docker exec "${CH_CONTAINER}" clickhouse-client \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" \ + aw_1c_clickhouse_client \ -q "SELECT if(count()=0, -1, dateDiff('hour', max(ts), now())) FROM ${table}" } diff --git a/clickhouse-1c/ops/clickhouse-client-safe.sh b/clickhouse-1c/ops/clickhouse-client-safe.sh new file mode 100644 index 0000000..587df92 --- /dev/null +++ b/clickhouse-1c/ops/clickhouse-client-safe.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +aw_1c_xml_escape() { + local value="$1" + value="${value//&/&}" + value="${value///>}" + value="${value//\"/"}" + value="${value//\'/'}" + printf '%s' "${value}" +} + +aw_1c_clickhouse_client() { + local container="${CH_CONTAINER:-${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}}" + local local_cfg remote_cfg status + + if [[ -z "${CLICKHOUSE_USER:-}" ]]; then + echo "CLICKHOUSE_USER is required" >&2 + return 1 + fi + if [[ -z "${CLICKHOUSE_DB:-}" ]]; then + echo "CLICKHOUSE_DB is required" >&2 + return 1 + fi + if [[ -z "${CLICKHOUSE_PASSWORD+x}" ]]; then + echo "CLICKHOUSE_PASSWORD is required" >&2 + return 1 + fi + + local_cfg="$(mktemp "${TMPDIR:-/tmp}/aw-1c-clickhouse-client.XXXXXX.xml")" + chmod 0600 "${local_cfg}" + remote_cfg="/tmp/aw-1c-clickhouse-client.$(date +%s).$$.xml" + + { + printf '\n' + printf ' %s\n' "$(aw_1c_xml_escape "${CLICKHOUSE_USER}")" + printf ' %s\n' "$(aw_1c_xml_escape "${CLICKHOUSE_PASSWORD}")" + printf ' %s\n' "$(aw_1c_xml_escape "${CLICKHOUSE_DB}")" + printf '\n' + } > "${local_cfg}" + + if ! docker exec -i "${container}" sh -c 'umask 077 && cat > "$1"' sh "${remote_cfg}" < "${local_cfg}"; then + rm -f "${local_cfg}" + echo "failed to stage ClickHouse client config in container" >&2 + return 1 + fi + rm -f "${local_cfg}" + + status=0 + docker exec -i "${container}" clickhouse-client --config-file "${remote_cfg}" "$@" || status=$? + docker exec "${container}" rm -f "${remote_cfg}" >/dev/null 2>&1 || true + return "${status}" +} diff --git a/clickhouse-1c/ops/run_company_intelligence_refresh.sh b/clickhouse-1c/ops/run_company_intelligence_refresh.sh index 19930de..032708a 100644 --- a/clickhouse-1c/ops/run_company_intelligence_refresh.sh +++ b/clickhouse-1c/ops/run_company_intelligence_refresh.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2119 set -euo pipefail ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" @@ -21,18 +22,20 @@ if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then exit 1 fi -# shellcheck disable=SC1090 +set -a +# shellcheck source=/dev/null . "${ENV_FILE}" +set +a +# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh +. "${ROOT}/ops/clickhouse-client-safe.sh" CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}" if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then CH_RUNTIME_HOST="127.0.0.1" fi +: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}" -docker exec -i "${CH_CONTAINER}" clickhouse-client \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" \ +aw_1c_clickhouse_client \ < "${ROOT}/clickhouse/init/04_company_intelligence.sql" "${ROOT}/ops/run_company_registry_bindings_refresh.sh" @@ -41,5 +44,4 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \ --host "${CH_RUNTIME_HOST}" \ --port "${CLICKHOUSE_PORT}" \ --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ --database "${CLICKHOUSE_DB}" diff --git a/clickhouse-1c/ops/run_company_registry_bindings_refresh.sh b/clickhouse-1c/ops/run_company_registry_bindings_refresh.sh index 9cefeeb..488833f 100644 --- a/clickhouse-1c/ops/run_company_registry_bindings_refresh.sh +++ b/clickhouse-1c/ops/run_company_registry_bindings_refresh.sh @@ -15,17 +15,19 @@ if [[ ! -x "${VENV}/bin/python" ]]; then exit 1 fi -# shellcheck disable=SC1090 +set -a +# shellcheck source=/dev/null . "${ENV_FILE}" +set +a CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}" if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then CH_RUNTIME_HOST="127.0.0.1" fi +: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}" "${VENV}/bin/python" "${ROOT}/ai/refresh_company_registry_bindings.py" \ --host "${CH_RUNTIME_HOST}" \ --port "${CLICKHOUSE_PORT}" \ --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ --database "${CLICKHOUSE_DB}" diff --git a/clickhouse-1c/ops/run_ingest_cycle.sh b/clickhouse-1c/ops/run_ingest_cycle.sh index 07a9f65..1fd9962 100644 --- a/clickhouse-1c/ops/run_ingest_cycle.sh +++ b/clickhouse-1c/ops/run_ingest_cycle.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2119 set -euo pipefail ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" @@ -40,6 +41,8 @@ fi # shellcheck disable=SC1090 . "${ENV_FILE}" +# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh +. "${ROOT}/ops/clickhouse-client-safe.sh" if [[ "${RUN_MCP_TOOLKIT_EXTRACT_BEFORE_INGEST}" == "1" ]]; then "${VENV}/bin/python" "${ROOT}/etl/extract_1c_mcp_toolkit.py" --config "${CONFIG}" @@ -49,31 +52,19 @@ fi "${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}" "${VENV}/bin/python" "${ROOT}/etl/load_company_registry_xlsx.py" --config "${CONFIG}" --landing "${ROOT}/landing/registry" -docker exec -i "${CH_CONTAINER}" clickhouse-client \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" \ +aw_1c_clickhouse_client \ < "${ROOT}/detections/build_entity_timeline.sql" -docker exec -i "${CH_CONTAINER}" clickhouse-client \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" \ +aw_1c_clickhouse_client \ < "${ROOT}/clickhouse/init/04_company_intelligence.sql" "${ROOT}/ops/run_company_registry_bindings_refresh.sh" "${ROOT}/ops/run_company_intelligence_refresh.sh" -docker exec -i "${CH_CONTAINER}" clickhouse-client \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" \ +aw_1c_clickhouse_client \ < "${ROOT}/detections/insert_detections.sql" -docker exec -i "${CH_CONTAINER}" clickhouse-client \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" \ +aw_1c_clickhouse_client \ < "${ROOT}/detections/open_cases_from_detections.sql" if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then diff --git a/clickhouse-1c/ops/run_manager_brief.sh b/clickhouse-1c/ops/run_manager_brief.sh index b6feedb..d1a3573 100755 --- a/clickhouse-1c/ops/run_manager_brief.sh +++ b/clickhouse-1c/ops/run_manager_brief.sh @@ -19,8 +19,8 @@ if [[ ! -x "${VENV}/bin/python" ]]; then exit 1 fi -# shellcheck disable=SC1090 set -a +# shellcheck source=/dev/null . "${ENV_FILE}" set +a @@ -28,6 +28,7 @@ CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}" if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then CH_RUNTIME_HOST="127.0.0.1" fi +: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}" export CLICKHOUSE_HOST="${CH_RUNTIME_HOST}" @@ -43,7 +44,6 @@ while (( attempt <= RETRIES )); do --host "${CH_RUNTIME_HOST}" \ --port "${CLICKHOUSE_PORT}" \ --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ --database "${CLICKHOUSE_DB}"; then exit 0 fi diff --git a/clickhouse-1c/ops/run_recovery_brief.sh b/clickhouse-1c/ops/run_recovery_brief.sh index 2deaeec..6d9be65 100644 --- a/clickhouse-1c/ops/run_recovery_brief.sh +++ b/clickhouse-1c/ops/run_recovery_brief.sh @@ -19,8 +19,8 @@ if [[ ! -x "${VENV}/bin/python" ]]; then exit 1 fi -# shellcheck disable=SC1090 set -a +# shellcheck source=/dev/null . "${ENV_FILE}" set +a @@ -28,6 +28,7 @@ CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}" if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then CH_RUNTIME_HOST="127.0.0.1" fi +: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}" exec 9>"${LOCK_FILE}" if ! flock -w "${LOCK_WAIT_SEC}" 9; then @@ -41,7 +42,6 @@ while (( attempt <= RETRIES )); do --host "${CH_RUNTIME_HOST}" \ --port "${CLICKHOUSE_PORT}" \ --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ --database "${CLICKHOUSE_DB}"; then exit 0 fi diff --git a/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md index 6651528..213983a 100644 --- a/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md +++ b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md @@ -228,3 +228,38 @@ powershell.exe -ExecutionPolicy Bypass ` Запрещено вручную удалять Windows collector queues, incident artifacts, DLP evidence, Hayabusa archives, Grafana data или ClickHouse tables без отдельного operator approval и backup/restore plan. + +## Проверка отсутствия ClickHouse-пароля в argv + +Цель проверки - убедиться, что ClickHouse/1C runtime wrappers не передают +`CLICKHOUSE_PASSWORD` через аргументы процессов. Пароль должен поступать из +`/opt/activitywatch/clickhouse-1c/.env` в окружение или временный client config, +а не через `--password`. + +Статическая проверка wrappers: + +```bash +rg -n -- '--password[= ]+"?\$[{]?CLICKHOUSE_PASSWORD' \ + /opt/activitywatch/clickhouse-1c/ops +``` + +Ожидаемый результат: команда не выводит совпадений. + +Runtime smoke во время ingest/brief refresh: + +```bash +set -a +. /opt/activitywatch/clickhouse-1c/.env +set +a + +ps -eo args= | grep -E 'clickhouse-client|generate_.*brief|refresh_company' | +while IFS= read -r line; do + if printf '%s' "${line}" | grep -F -- "${CLICKHOUSE_PASSWORD}" >/dev/null; then + echo "FAIL: ClickHouse password is visible in process argv" >&2 + exit 1 + fi +done +``` + +Ожидаемый результат: команда завершается с кодом `0` и не печатает секрет. +Если проверка падает, остановить rollout и вернуть предыдущий release artifact.