462 lines
13 KiB
Rust
462 lines
13 KiB
Rust
use std::net::{TcpStream, ToSocketAddrs};
|
|
use std::process::Command;
|
|
use std::time::Duration;
|
|
|
|
use anyhow::{Context, Result};
|
|
use clap::{Parser, ValueEnum};
|
|
use reqwest::blocking::Client;
|
|
use reqwest::redirect::Policy;
|
|
use serde_json::Value;
|
|
|
|
#[derive(Debug, Parser)]
|
|
#[command(about = "ActivityWatch-Russian contour smoke checks")]
|
|
struct Cli {
|
|
#[arg(long, value_enum, default_value_t = Mode::ProxmoxRemote)]
|
|
mode: Mode,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, ValueEnum)]
|
|
enum Mode {
|
|
ProxmoxRemote,
|
|
}
|
|
|
|
#[derive(Debug, Default)]
|
|
struct Counts {
|
|
ok: usize,
|
|
warn: usize,
|
|
fail: usize,
|
|
skip: usize,
|
|
}
|
|
|
|
impl Counts {
|
|
fn pass(&mut self, msg: impl AsRef<str>) {
|
|
self.ok += 1;
|
|
println!("[OK] {}", msg.as_ref());
|
|
}
|
|
|
|
fn fail(&mut self, msg: impl AsRef<str>) {
|
|
self.fail += 1;
|
|
println!("[FAIL] {}", msg.as_ref());
|
|
}
|
|
|
|
fn skip(&mut self, msg: impl AsRef<str>) {
|
|
self.skip += 1;
|
|
println!("[SKIP] {}", msg.as_ref());
|
|
}
|
|
}
|
|
|
|
fn main() {
|
|
let code = match run() {
|
|
Ok(code) => code,
|
|
Err(err) => {
|
|
eprintln!("{err:#}");
|
|
1
|
|
}
|
|
};
|
|
std::process::exit(code);
|
|
}
|
|
|
|
fn run() -> Result<i32> {
|
|
let cli = Cli::parse();
|
|
match cli.mode {
|
|
Mode::ProxmoxRemote => run_proxmox_remote(),
|
|
}
|
|
}
|
|
|
|
fn run_proxmox_remote() -> Result<i32> {
|
|
let mut counts = Counts::default();
|
|
let http = Client::builder()
|
|
.no_proxy()
|
|
.danger_accept_invalid_certs(true)
|
|
.timeout(Duration::from_secs(15))
|
|
.build()
|
|
.context("build HTTP client")?;
|
|
let no_redirect_http = Client::builder()
|
|
.no_proxy()
|
|
.danger_accept_invalid_certs(true)
|
|
.timeout(Duration::from_secs(15))
|
|
.redirect(Policy::none())
|
|
.build()
|
|
.context("build no-redirect HTTP client")?;
|
|
|
|
section("Host");
|
|
print_command("hostnamectl", &["hostnamectl"]);
|
|
print_command("date", &["date", "-Is"]);
|
|
print_command("uptime", &["uptime"]);
|
|
|
|
section("Core Services");
|
|
for unit in [
|
|
"nginx.service",
|
|
"pveproxy.service",
|
|
"pvedaemon.service",
|
|
"pvestatd.service",
|
|
"pve-cluster.service",
|
|
"docker.service",
|
|
"aw-1c-company-api.service",
|
|
"aw-pve-webadmin-logger.service",
|
|
] {
|
|
check_service(&mut counts, unit);
|
|
}
|
|
|
|
section("Timers");
|
|
for unit in [
|
|
"aw-1c-ingest.timer",
|
|
"aw-1c-proofcheck.timer",
|
|
"aw-1c-manager-brief.timer",
|
|
"aw-1c-recovery-brief.timer",
|
|
"aw-1c-weekly-digest.timer",
|
|
] {
|
|
check_timer(&mut counts, unit);
|
|
}
|
|
if let Ok(out) = command_output("systemctl", &["list-timers", "--all", "--no-pager"]) {
|
|
print_filtered_lines(&out, &["aw-1c", "NEXT", "LEFT", "PASSED"], 40);
|
|
}
|
|
|
|
section("Ports");
|
|
check_tcp(&mut counts, "nginx http", "127.0.0.1", 80);
|
|
check_tcp(&mut counts, "nginx https", "127.0.0.1", 443);
|
|
check_tcp(&mut counts, "proxmox web", "127.0.0.1", 8006);
|
|
check_tcp(&mut counts, "1C company API", "192.0.2.2", 8710);
|
|
check_tcp(&mut counts, "clickhouse native", "127.0.0.1", 9000);
|
|
check_tcp(&mut counts, "clickhouse http", "127.0.0.1", 8123);
|
|
if let Ok(out) = command_output("ss", &["-tulpn"]) {
|
|
print_filtered_lines(
|
|
&out,
|
|
&[":80", ":443", ":8006", ":8710", ":8123", ":9000"],
|
|
40,
|
|
);
|
|
}
|
|
|
|
section("Gateway HTTP");
|
|
check_http_code(
|
|
&mut counts,
|
|
&http,
|
|
"nginx healthz",
|
|
"https://127.0.0.1/healthz",
|
|
&[200],
|
|
);
|
|
check_http_redirect(
|
|
&mut counts,
|
|
&no_redirect_http,
|
|
"go proxmox gui",
|
|
"https://127.0.0.1/go/proxmox-gui",
|
|
&[301, 302, 307, 308],
|
|
Some("https://192.0.2.2:8006/"),
|
|
);
|
|
check_http_redirect(
|
|
&mut counts,
|
|
&no_redirect_http,
|
|
"go file1c brief",
|
|
"https://127.0.0.1/go/file1c-brief",
|
|
&[301, 302, 307, 308],
|
|
Some("http://192.0.2.2:8710/manager/brief"),
|
|
);
|
|
check_http_redirect(
|
|
&mut counts,
|
|
&no_redirect_http,
|
|
"go file1c actions",
|
|
"https://127.0.0.1/go/file1c-actions",
|
|
&[301, 302, 307, 308],
|
|
Some("http://192.0.2.2:8710/manager/actions"),
|
|
);
|
|
|
|
section("1C Company API");
|
|
check_http_code(
|
|
&mut counts,
|
|
&no_redirect_http,
|
|
"1C root redirect",
|
|
"http://192.0.2.2:8710/",
|
|
&[307],
|
|
);
|
|
for (name, url) in [
|
|
("1C /health", "http://192.0.2.2:8710/health"),
|
|
("1C /api/health", "http://192.0.2.2:8710/api/health"),
|
|
("1C manager brief", "http://192.0.2.2:8710/manager/brief"),
|
|
(
|
|
"1C manager actions",
|
|
"http://192.0.2.2:8710/manager/actions",
|
|
),
|
|
(
|
|
"1C manager recovery",
|
|
"http://192.0.2.2:8710/manager/recovery",
|
|
),
|
|
(
|
|
"1C weekly digest",
|
|
"http://192.0.2.2:8710/manager/digest/weekly",
|
|
),
|
|
] {
|
|
check_http_code(&mut counts, &http, name, url, &[200]);
|
|
}
|
|
|
|
section("ClickHouse");
|
|
check_docker_container(&mut counts, "aw-rus-1c-clickhouse");
|
|
check_http_code(
|
|
&mut counts,
|
|
&http,
|
|
"ClickHouse ping",
|
|
"http://127.0.0.1:8123/ping",
|
|
&[200],
|
|
);
|
|
if command_exists("docker") && docker_container_running("aw-rus-1c-clickhouse") {
|
|
check_command(
|
|
&mut counts,
|
|
"ClickHouse SELECT 1",
|
|
"docker",
|
|
&[
|
|
"exec",
|
|
"aw-rus-1c-clickhouse",
|
|
"clickhouse-client",
|
|
"--query",
|
|
"SELECT 1",
|
|
],
|
|
);
|
|
}
|
|
|
|
section("System Capacity");
|
|
print_command("df", &["df", "-h", "/", "/var", "/opt"]);
|
|
print_command("free", &["free", "-h"]);
|
|
|
|
section("Summary");
|
|
println!(
|
|
"OK={} WARN={} FAIL={} SKIP={}",
|
|
counts.ok, counts.warn, counts.fail, counts.skip
|
|
);
|
|
Ok(if counts.fail > 0 { 2 } else { 0 })
|
|
}
|
|
|
|
fn section(name: &str) {
|
|
println!();
|
|
println!("== {name} ==");
|
|
}
|
|
|
|
fn command_exists(cmd: &str) -> bool {
|
|
Command::new("sh")
|
|
.arg("-c")
|
|
.arg(format!("command -v {cmd} >/dev/null 2>&1"))
|
|
.status()
|
|
.map(|status| status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn command_output(cmd: &str, args: &[&str]) -> Result<String> {
|
|
let output = Command::new(cmd)
|
|
.args(args)
|
|
.output()
|
|
.with_context(|| format!("run {cmd}"))?;
|
|
let mut text = String::new();
|
|
text.push_str(&String::from_utf8_lossy(&output.stdout));
|
|
text.push_str(&String::from_utf8_lossy(&output.stderr));
|
|
if output.status.success() {
|
|
Ok(text)
|
|
} else {
|
|
Err(anyhow::anyhow!("{cmd} failed: {text}"))
|
|
}
|
|
}
|
|
|
|
fn print_command(label: &str, command: &[&str]) {
|
|
if let Some((cmd, args)) = command.split_first() {
|
|
match command_output(cmd, args) {
|
|
Ok(out) => print_indented(&out, 80),
|
|
Err(err) => println!(" {label}: {err:#}"),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn print_indented(text: &str, max_lines: usize) {
|
|
for line in text.lines().take(max_lines) {
|
|
println!(" {line}");
|
|
}
|
|
}
|
|
|
|
fn print_filtered_lines(text: &str, patterns: &[&str], max_lines: usize) {
|
|
for line in text
|
|
.lines()
|
|
.filter(|line| patterns.iter().any(|pattern| line.contains(pattern)))
|
|
.take(max_lines)
|
|
{
|
|
println!(" {line}");
|
|
}
|
|
}
|
|
|
|
fn check_service(counts: &mut Counts, unit: &str) {
|
|
check_systemd_unit(counts, unit, "service");
|
|
}
|
|
|
|
fn check_timer(counts: &mut Counts, unit: &str) {
|
|
check_systemd_unit(counts, unit, "timer");
|
|
}
|
|
|
|
fn check_systemd_unit(counts: &mut Counts, unit: &str, kind: &str) {
|
|
if !Command::new("systemctl")
|
|
.args(["list-unit-files", unit])
|
|
.status()
|
|
.map(|status| status.success())
|
|
.unwrap_or(false)
|
|
{
|
|
counts.skip(format!("{unit} is not installed"));
|
|
return;
|
|
}
|
|
if Command::new("systemctl")
|
|
.args(["is-active", "--quiet", unit])
|
|
.status()
|
|
.map(|status| status.success())
|
|
.unwrap_or(false)
|
|
{
|
|
counts.pass(format!("{unit} active"));
|
|
} else {
|
|
counts.fail(format!("{unit} inactive or failed"));
|
|
if let Ok(out) = command_output("systemctl", &["--no-pager", "--lines=8", "status", unit]) {
|
|
print_indented(&out, 30);
|
|
} else {
|
|
let _ = kind;
|
|
}
|
|
}
|
|
}
|
|
|
|
fn check_tcp(counts: &mut Counts, name: &str, host: &str, port: u16) {
|
|
let addr = format!("{host}:{port}");
|
|
let ok = addr
|
|
.to_socket_addrs()
|
|
.ok()
|
|
.and_then(|mut addrs| addrs.next())
|
|
.and_then(|addr| TcpStream::connect_timeout(&addr, Duration::from_secs(4)).ok())
|
|
.is_some();
|
|
if ok {
|
|
counts.pass(format!("{name} TCP {host}:{port}"));
|
|
} else {
|
|
counts.fail(format!("{name} TCP {host}:{port}"));
|
|
}
|
|
}
|
|
|
|
fn check_http_code(counts: &mut Counts, client: &Client, name: &str, url: &str, expected: &[u16]) {
|
|
match client.get(url).send() {
|
|
Ok(response) => {
|
|
let code = response.status().as_u16();
|
|
if expected.contains(&code) {
|
|
counts.pass(format!("{name} HTTP {code} {url}"));
|
|
} else {
|
|
counts.fail(format!("{name} HTTP {code} {url}"));
|
|
if let Ok(text) = response.text() {
|
|
print_indented(&text, 40);
|
|
}
|
|
}
|
|
}
|
|
Err(err) => counts.fail(format!("{name} HTTP error {url}: {err}")),
|
|
}
|
|
}
|
|
|
|
fn check_http_redirect(
|
|
counts: &mut Counts,
|
|
client: &Client,
|
|
name: &str,
|
|
url: &str,
|
|
expected: &[u16],
|
|
expected_location: Option<&str>,
|
|
) {
|
|
match client.head(url).send() {
|
|
Ok(response) => {
|
|
let code = response.status().as_u16();
|
|
let location = response
|
|
.headers()
|
|
.get(reqwest::header::LOCATION)
|
|
.and_then(|value| value.to_str().ok())
|
|
.unwrap_or("");
|
|
let location_ok = expected_location.is_none_or(|expected| location.contains(expected));
|
|
if expected.contains(&code) && location_ok {
|
|
counts.pass(format!(
|
|
"{name} HTTP {code} {}",
|
|
if location.is_empty() { url } else { location }
|
|
));
|
|
} else {
|
|
counts.fail(format!(
|
|
"{name} HTTP {code} {}",
|
|
if location.is_empty() { url } else { location }
|
|
));
|
|
}
|
|
}
|
|
Err(err) => counts.fail(format!("{name} HTTP error {url}: {err}")),
|
|
}
|
|
}
|
|
|
|
fn check_command(counts: &mut Counts, name: &str, cmd: &str, args: &[&str]) {
|
|
match command_output(cmd, args) {
|
|
Ok(out) => {
|
|
counts.pass(name);
|
|
print_indented(&out, 40);
|
|
}
|
|
Err(err) => counts.fail(format!("{name}: {err:#}")),
|
|
}
|
|
}
|
|
|
|
fn check_docker_container(counts: &mut Counts, name: &str) {
|
|
if !command_exists("docker") {
|
|
counts.skip("docker command unavailable");
|
|
return;
|
|
}
|
|
if docker_container_running(name) {
|
|
counts.pass(format!("docker container {name} running"));
|
|
if let Ok(out) = command_output(
|
|
"docker",
|
|
&[
|
|
"ps",
|
|
"--filter",
|
|
&format!("name=^/{name}$"),
|
|
"--format",
|
|
"{{.Names}} {{.Status}} {{.Ports}}",
|
|
],
|
|
) {
|
|
print_indented(&out, 20);
|
|
}
|
|
} else {
|
|
counts.fail(format!("docker container {name} not running"));
|
|
if let Ok(out) = command_output(
|
|
"docker",
|
|
&[
|
|
"ps",
|
|
"-a",
|
|
"--filter",
|
|
&format!("name=^/{name}$"),
|
|
"--format",
|
|
"{{.Names}} {{.Status}} {{.Ports}}",
|
|
],
|
|
) {
|
|
print_indented(&out, 20);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn docker_container_running(name: &str) -> bool {
|
|
command_output("docker", &["ps", "--format", "{{.Names}}"])
|
|
.map(|out| out.lines().any(|line| line == name))
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
fn parse_json_key_present(value: &Value, key: &str) -> bool {
|
|
value.get(key).is_some()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn counts_exit_code_matches_failures() {
|
|
let counts = Counts {
|
|
ok: 1,
|
|
warn: 1,
|
|
fail: 0,
|
|
skip: 1,
|
|
};
|
|
assert_eq!(if counts.fail > 0 { 2 } else { 0 }, 0);
|
|
let counts = Counts { fail: 1, ..counts };
|
|
assert_eq!(if counts.fail > 0 { 2 } else { 0 }, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn default_counts_are_zero() {
|
|
let counts = Counts::default();
|
|
assert_eq!(counts.ok + counts.warn + counts.fail + counts.skip, 0);
|
|
}
|
|
}
|