feat(workforce): add ClickHouse workforce analytics contour
This commit is contained in:
igor04091968
@@ -0,0 +1 @@
|
||||
CREATE DATABASE IF NOT EXISTS aw_workforce;
|
||||
@@ -0,0 +1,32 @@
|
||||
CREATE TABLE IF NOT EXISTS aw_workforce.aw_window_events
|
||||
(
|
||||
event_time DateTime,
|
||||
host_name String,
|
||||
user_login String,
|
||||
process_name String,
|
||||
window_title String,
|
||||
duration_sec UInt32,
|
||||
source_bucket LowCardinality(String),
|
||||
source_event_id String,
|
||||
ingested_at DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(event_time)
|
||||
ORDER BY (event_time, host_name, user_login, process_name, source_event_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS aw_workforce.aw_browser_events
|
||||
(
|
||||
event_time DateTime,
|
||||
host_name String,
|
||||
user_login String,
|
||||
browser_name String,
|
||||
url String,
|
||||
title String,
|
||||
duration_sec UInt32,
|
||||
source_bucket LowCardinality(String),
|
||||
source_event_id String,
|
||||
ingested_at DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(event_time)
|
||||
ORDER BY (event_time, host_name, user_login, browser_name, source_event_id);
|
||||
@@ -0,0 +1,145 @@
|
||||
CREATE TABLE IF NOT EXISTS aw_workforce.dim_workstation_user
|
||||
(
|
||||
host_name String,
|
||||
user_login String,
|
||||
user_domain String,
|
||||
|
||||
employee_id String,
|
||||
employee_name String,
|
||||
department String,
|
||||
branch String,
|
||||
position String,
|
||||
|
||||
source LowCardinality(String),
|
||||
is_active UInt8 DEFAULT 1,
|
||||
updated_at DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(updated_at)
|
||||
ORDER BY (host_name, user_login);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS aw_workforce.dim_application_category
|
||||
(
|
||||
process_name String,
|
||||
|
||||
application_name String,
|
||||
vendor String,
|
||||
category LowCardinality(String),
|
||||
productivity_class LowCardinality(String),
|
||||
risk_level LowCardinality(String),
|
||||
|
||||
is_system UInt8 DEFAULT 0,
|
||||
is_active UInt8 DEFAULT 1,
|
||||
source LowCardinality(String),
|
||||
comment String,
|
||||
updated_at DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(updated_at)
|
||||
ORDER BY process_name;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS aw_workforce.dim_domain_category
|
||||
(
|
||||
domain String,
|
||||
|
||||
site_name String,
|
||||
category LowCardinality(String),
|
||||
productivity_class LowCardinality(String),
|
||||
risk_level LowCardinality(String),
|
||||
business_allowed UInt8 DEFAULT 0,
|
||||
|
||||
source LowCardinality(String),
|
||||
comment String,
|
||||
is_active UInt8 DEFAULT 1,
|
||||
updated_at DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(updated_at)
|
||||
ORDER BY domain;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS aw_workforce.dim_url_rule
|
||||
(
|
||||
rule_id String,
|
||||
domain String,
|
||||
path_pattern String,
|
||||
|
||||
category LowCardinality(String),
|
||||
productivity_class LowCardinality(String),
|
||||
risk_level LowCardinality(String),
|
||||
|
||||
priority UInt16 DEFAULT 100,
|
||||
is_active UInt8 DEFAULT 1,
|
||||
comment String,
|
||||
updated_at DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(updated_at)
|
||||
ORDER BY (domain, priority, rule_id);
|
||||
|
||||
DROP DICTIONARY IF EXISTS aw_workforce.dict_workstation_user;
|
||||
|
||||
CREATE DICTIONARY aw_workforce.dict_workstation_user
|
||||
(
|
||||
host_name String,
|
||||
user_login String,
|
||||
user_domain String,
|
||||
|
||||
employee_id String,
|
||||
employee_name String,
|
||||
department String,
|
||||
branch String,
|
||||
position String,
|
||||
is_active UInt8
|
||||
)
|
||||
PRIMARY KEY host_name, user_login
|
||||
SOURCE(CLICKHOUSE(
|
||||
USER 'aw_workforce_dict'
|
||||
PASSWORD ''
|
||||
DB 'aw_workforce'
|
||||
TABLE 'dim_workstation_user'
|
||||
))
|
||||
LAYOUT(COMPLEX_KEY_HASHED())
|
||||
LIFETIME(MIN 3600 MAX 86400);
|
||||
|
||||
DROP DICTIONARY IF EXISTS aw_workforce.dict_application_category;
|
||||
|
||||
CREATE DICTIONARY aw_workforce.dict_application_category
|
||||
(
|
||||
process_name String,
|
||||
|
||||
application_name String,
|
||||
vendor String,
|
||||
category String,
|
||||
productivity_class String,
|
||||
risk_level String,
|
||||
is_system UInt8,
|
||||
is_active UInt8
|
||||
)
|
||||
PRIMARY KEY process_name
|
||||
SOURCE(CLICKHOUSE(
|
||||
USER 'aw_workforce_dict'
|
||||
PASSWORD ''
|
||||
DB 'aw_workforce'
|
||||
TABLE 'dim_application_category'
|
||||
))
|
||||
LAYOUT(HASHED())
|
||||
LIFETIME(MIN 3600 MAX 86400);
|
||||
|
||||
DROP DICTIONARY IF EXISTS aw_workforce.dict_domain_category;
|
||||
|
||||
CREATE DICTIONARY aw_workforce.dict_domain_category
|
||||
(
|
||||
domain String,
|
||||
|
||||
site_name String,
|
||||
category String,
|
||||
productivity_class String,
|
||||
risk_level String,
|
||||
business_allowed UInt8,
|
||||
is_active UInt8
|
||||
)
|
||||
PRIMARY KEY domain
|
||||
SOURCE(CLICKHOUSE(
|
||||
USER 'aw_workforce_dict'
|
||||
PASSWORD ''
|
||||
DB 'aw_workforce'
|
||||
TABLE 'dim_domain_category'
|
||||
))
|
||||
LAYOUT(HASHED())
|
||||
LIFETIME(MIN 3600 MAX 86400);
|
||||
@@ -0,0 +1,158 @@
|
||||
CREATE TABLE IF NOT EXISTS aw_workforce.agg_workforce_productivity_hourly
|
||||
(
|
||||
bucket_start DateTime,
|
||||
event_date Date,
|
||||
|
||||
branch LowCardinality(String),
|
||||
department LowCardinality(String),
|
||||
|
||||
activity_type LowCardinality(String),
|
||||
category LowCardinality(String),
|
||||
productivity_class LowCardinality(String),
|
||||
|
||||
duration_sec UInt64,
|
||||
event_count UInt64,
|
||||
unknown_subject_events UInt64,
|
||||
unknown_category_events UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree((
|
||||
duration_sec,
|
||||
event_count,
|
||||
unknown_subject_events,
|
||||
unknown_category_events
|
||||
))
|
||||
PARTITION BY toYYYYMM(event_date)
|
||||
ORDER BY (
|
||||
event_date,
|
||||
bucket_start,
|
||||
branch,
|
||||
department,
|
||||
activity_type,
|
||||
productivity_class,
|
||||
category
|
||||
);
|
||||
|
||||
DROP VIEW IF EXISTS aw_workforce.mv_desktop_productivity_hourly;
|
||||
|
||||
CREATE MATERIALIZED VIEW aw_workforce.mv_desktop_productivity_hourly
|
||||
TO aw_workforce.agg_workforce_productivity_hourly
|
||||
AS
|
||||
SELECT
|
||||
toStartOfHour(event_time) AS bucket_start,
|
||||
toDate(event_time) AS event_date,
|
||||
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'),
|
||||
'unknown'
|
||||
) AS branch,
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'),
|
||||
'unknown'
|
||||
) AS department,
|
||||
|
||||
'desktop' AS activity_type,
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, 'unknown'),
|
||||
'unknown'
|
||||
) AS category,
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, 'unknown'),
|
||||
'unknown'
|
||||
) AS productivity_class,
|
||||
|
||||
toUInt64(sum(duration_sec)) AS duration_sec,
|
||||
toUInt64(count()) AS event_count,
|
||||
toUInt64(sum(if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '',
|
||||
0,
|
||||
1
|
||||
))) AS unknown_subject_events,
|
||||
toUInt64(sum(if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') NOT IN ('', 'unknown'),
|
||||
0,
|
||||
1
|
||||
))) AS unknown_category_events
|
||||
FROM aw_workforce.aw_window_events
|
||||
GROUP BY
|
||||
bucket_start,
|
||||
event_date,
|
||||
branch,
|
||||
department,
|
||||
activity_type,
|
||||
category,
|
||||
productivity_class;
|
||||
|
||||
DROP VIEW IF EXISTS aw_workforce.mv_browser_productivity_hourly;
|
||||
|
||||
CREATE MATERIALIZED VIEW aw_workforce.mv_browser_productivity_hourly
|
||||
TO aw_workforce.agg_workforce_productivity_hourly
|
||||
AS
|
||||
WITH
|
||||
lowerUTF8(
|
||||
domain(if(position(url, '://') = 0, concat('http://', url), url))
|
||||
) AS domain_name
|
||||
SELECT
|
||||
toStartOfHour(event_time) AS bucket_start,
|
||||
toDate(event_time) AS event_date,
|
||||
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'),
|
||||
'unknown'
|
||||
) AS branch,
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'),
|
||||
'unknown'
|
||||
) AS department,
|
||||
|
||||
'browser' AS activity_type,
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, 'unknown'),
|
||||
'unknown'
|
||||
) AS category,
|
||||
if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, '') != '',
|
||||
dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, 'unknown'),
|
||||
'unknown'
|
||||
) AS productivity_class,
|
||||
|
||||
toUInt64(sum(duration_sec)) AS duration_sec,
|
||||
toUInt64(count()) AS event_count,
|
||||
toUInt64(sum(if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '',
|
||||
0,
|
||||
1
|
||||
))) AS unknown_subject_events,
|
||||
toUInt64(sum(if(
|
||||
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
|
||||
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') NOT IN ('', 'unknown'),
|
||||
0,
|
||||
1
|
||||
))) AS unknown_category_events
|
||||
FROM aw_workforce.aw_browser_events
|
||||
WHERE domain_name != ''
|
||||
GROUP BY
|
||||
bucket_start,
|
||||
event_date,
|
||||
branch,
|
||||
department,
|
||||
activity_type,
|
||||
category,
|
||||
productivity_class;
|
||||
@@ -0,0 +1,105 @@
|
||||
CREATE OR REPLACE VIEW aw_workforce.v_workforce_productivity_daily AS
|
||||
SELECT
|
||||
event_date,
|
||||
branch,
|
||||
department,
|
||||
activity_type,
|
||||
category,
|
||||
productivity_class,
|
||||
sum(duration_sec) AS duration_sec,
|
||||
sum(event_count) AS event_count,
|
||||
sum(unknown_subject_events) AS unknown_subject_events,
|
||||
sum(unknown_category_events) AS unknown_category_events
|
||||
FROM aw_workforce.agg_workforce_productivity_hourly
|
||||
GROUP BY
|
||||
event_date,
|
||||
branch,
|
||||
department,
|
||||
activity_type,
|
||||
category,
|
||||
productivity_class;
|
||||
|
||||
CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_subjects AS
|
||||
SELECT
|
||||
host_name,
|
||||
user_login,
|
||||
count() AS events,
|
||||
sum(duration_sec) AS duration_sec
|
||||
FROM aw_workforce.aw_window_events
|
||||
WHERE dictGetUInt8OrDefault(
|
||||
'aw_workforce.dict_workstation_user',
|
||||
'is_active',
|
||||
(host_name, user_login),
|
||||
0
|
||||
) != 1
|
||||
OR dictGetStringOrDefault(
|
||||
'aw_workforce.dict_workstation_user',
|
||||
'employee_name',
|
||||
(host_name, user_login),
|
||||
''
|
||||
) = ''
|
||||
GROUP BY
|
||||
host_name,
|
||||
user_login
|
||||
ORDER BY duration_sec DESC;
|
||||
|
||||
CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_processes AS
|
||||
SELECT
|
||||
process_name,
|
||||
count() AS events,
|
||||
sum(duration_sec) AS duration_sec
|
||||
FROM aw_workforce.aw_window_events
|
||||
WHERE dictGetUInt8OrDefault(
|
||||
'aw_workforce.dict_application_category',
|
||||
'is_active',
|
||||
process_name,
|
||||
0
|
||||
) != 1
|
||||
OR dictGetStringOrDefault(
|
||||
'aw_workforce.dict_application_category',
|
||||
'category',
|
||||
process_name,
|
||||
''
|
||||
) IN ('', 'unknown')
|
||||
GROUP BY process_name
|
||||
ORDER BY duration_sec DESC;
|
||||
|
||||
CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_domains AS
|
||||
WITH
|
||||
lowerUTF8(
|
||||
domain(if(position(url, '://') = 0, concat('http://', url), url))
|
||||
) AS domain_name
|
||||
SELECT
|
||||
domain_name,
|
||||
count() AS events,
|
||||
sum(duration_sec) AS duration_sec
|
||||
FROM aw_workforce.aw_browser_events
|
||||
WHERE domain_name != ''
|
||||
AND (
|
||||
dictGetUInt8OrDefault(
|
||||
'aw_workforce.dict_domain_category',
|
||||
'is_active',
|
||||
domain_name,
|
||||
0
|
||||
) != 1
|
||||
OR dictGetStringOrDefault(
|
||||
'aw_workforce.dict_domain_category',
|
||||
'category',
|
||||
domain_name,
|
||||
''
|
||||
) IN ('', 'unknown')
|
||||
)
|
||||
GROUP BY domain_name
|
||||
ORDER BY duration_sec DESC;
|
||||
|
||||
CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_quality_daily AS
|
||||
SELECT
|
||||
event_date,
|
||||
sum(event_count) AS events,
|
||||
sum(unknown_subject_events) AS unknown_subject_events,
|
||||
round(unknown_subject_events / nullIf(events, 0), 4) AS unknown_subject_ratio,
|
||||
sum(unknown_category_events) AS unknown_category_events,
|
||||
round(unknown_category_events / nullIf(events, 0), 4) AS unknown_category_ratio
|
||||
FROM aw_workforce.agg_workforce_productivity_hourly
|
||||
GROUP BY event_date
|
||||
ORDER BY event_date DESC;
|
||||
@@ -0,0 +1,18 @@
|
||||
<clickhouse>
|
||||
<users>
|
||||
<default>
|
||||
<networks replace="replace">
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
</default>
|
||||
<aw_workforce_dict>
|
||||
<no_password/>
|
||||
<networks>
|
||||
<ip>127.0.0.1</ip>
|
||||
<ip>::1</ip>
|
||||
</networks>
|
||||
<profile>readonly</profile>
|
||||
<quota>default</quota>
|
||||
</aw_workforce_dict>
|
||||
</users>
|
||||
</clickhouse>
|
||||
Reference in New Issue
Block a user