diff --git a/.env b/.env new file mode 100644 index 0000000..03ba094 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +CLICKHOUSE_USER=default +CLICKHOUSE_PASSWORD=default +CLICKHOUSE_DB=default +CLICKHOUSE_PORT=8123 diff --git a/Makefile b/Makefile index c57a52c..aeb8bc8 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,24 @@ +include .env SHELL = /bin/bash -o pipefail ROOT_DIR=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -MIGRATIONS_DIR=$(SRC_DIR)/files/migrations +MIGRATIONS_DIR=$(ROOT_DIR)/files/migrations + +all: migrator migrate up + +migrator: + @docker buildx build --tag neur0toxine/clickhouse-migrations:latest -f $(ROOT_DIR)/files/Dockerfile.migrator $(ROOT_DIR)/files migrate: @docker-compose up -d - @docker run -v $(MIGRATIONS_DIR):/migrations --network host migrate/migrate -path=/migrations/ -database "clickhouse://localhost:8123?username=default&password=default&database=default" up all + @docker run -v $(MIGRATIONS_DIR):/migrations --network host neur0toxine/clickhouse-migrations:latest migrate --host=http://localhost:$(CLICKHOUSE_PORT) --user=$(CLICKHOUSE_USER) --password=$(CLICKHOUSE_PASSWORD) --db=default --migrations-home=/migrations @docker-compose stop up: @docker-compose up + +stop: + @docker-compose stop + +down: + @docker-compose down -v diff --git a/dashboards/Logs + Structured via Query (zaplog).json b/dashboards/Logs + Structured via Query (zaplog).json new file mode 100644 index 0000000..1f67cf0 --- /dev/null +++ b/dashboards/Logs + Structured via Query (zaplog).json @@ -0,0 +1,1063 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "panels": [], + "title": "Total & Errors", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "recps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "Europe/Moscow" + ], + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "priority", + "logdatetime" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [ + { + "aggregation": "count", + "alias": "", + "field": "" + } + ], + "mode": "trend", + "orderBy": [], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", + "refId": "A" + } + ], + "title": "Total messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "priority" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "priority", + "operator": "=", + "type": "String", + "value": "error" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "trend", + "orderBy": [], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", + "refId": "A" + } + ], + "title": "Total errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "panels": [], + "title": "Structured logs", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "displayName": "Priority", + "mappings": [ + { + "options": { + "DEBUG": { + "color": "dark-green", + "index": 0 + }, + "ERROR": { + "color": "dark-red", + "index": 3 + }, + "INFO": { + "color": "dark-blue", + "index": 1 + }, + "WARN": { + "color": "dark-yellow", + "index": 2 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "string", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "logdatetime" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "program" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time" + }, + "properties": [ + { + "id": "custom.width", + "value": 266 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "level" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "handler" + }, + "properties": [ + { + "id": "custom.width", + "value": 217 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "msg" + }, + "properties": [ + { + "id": "custom.width", + "value": 1307 + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Message" + }, + "properties": [ + { + "id": "custom.width", + "value": 639 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Context" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "connection" + }, + "properties": [ + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "account" + }, + "properties": [ + { + "id": "custom.width", + "value": 223 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Context" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Message" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Priority" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Handler" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Priority" + } + ] + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "table": "log_docker_raw" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "database": "default", + "fields": [ + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "table": "log_docker_raw" + } + }, + "queryType": "sql", + "rawSql": "SELECT\n visitParamExtractString(message, 'timestamp') as time,\n visitParamExtractString(message, 'level') as level,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'message') as msg,\n visitParamExtractRaw(message, 'context') as context\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000", + "refId": "A", + "selectedFormat": 4 + } + ], + "title": "Structured logs output", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "account": "Account", + "connection": "Connection", + "context": "Context", + "handler": "Handler", + "level": "Priority", + "msg": "Message", + "raw": "Raw", + "time": "TS" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 1, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "logdatetime" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "program" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "message" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "host", + "program", + "logdatetime", + "priority", + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "logdatetime" + } + ], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "builder", + "rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"priority\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000", + "refId": "A", + "selectedFormat": 4 + } + ], + "title": "Panel Title", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "telegram-mg_telegram-1", + "value": "telegram-mg_telegram-1" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b98b9382-2c99-41c0-bb27-dfd7033d71fa" + }, + "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", + "hide": 0, + "includeAll": false, + "label": "Program", + "multi": false, + "name": "program", + "options": [], + "query": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": [ + "Any" + ], + "value": [ + "Any" + ] + }, + "hide": 0, + "includeAll": false, + "label": "Priority", + "multi": true, + "name": "priority", + "options": [ + { + "selected": true, + "text": "Any", + "value": "Any" + }, + { + "selected": false, + "text": "DEBUG", + "value": "DEBUG" + }, + { + "selected": false, + "text": "INFO", + "value": "INFO" + }, + { + "selected": false, + "text": "WARN", + "value": "WARN" + }, + { + "selected": false, + "text": "ERROR", + "value": "ERROR" + } + ], + "query": "Any,DEBUG,INFO,WARN,ERROR", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter #1", + "name": "filter1", + "options": [ + { + "selected": true, + "text": "*", + "value": "*" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": {}, + "hide": 0, + "name": "filter2", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": {}, + "hide": 0, + "name": "filter3", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Logs + Structured via Query (zaplog)", + "uid": "b445eb43-25d1-4112-b628-3b8acf037f10", + "version": 6, + "weekStart": "" +} \ No newline at end of file diff --git a/files/Dockerfile.migrator b/files/Dockerfile.migrator new file mode 100644 index 0000000..3d43416 --- /dev/null +++ b/files/Dockerfile.migrator @@ -0,0 +1,3 @@ +FROM node:lts-alpine +RUN npm install --global clickhouse-migrations && npm cache clean --force +ENTRYPOINT ["/usr/local/bin/clickhouse-migrations"] \ No newline at end of file diff --git a/files/migrations/1698135711_create_logs_table.down.sql b/files/migrations/1698135711_create_logs_table.down.sql deleted file mode 100644 index b1c111a..0000000 --- a/files/migrations/1698135711_create_logs_table.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE log_docker_raw; \ No newline at end of file diff --git a/files/migrations/1698135711_create_logs_table.up.sql b/files/migrations/1_init.sql similarity index 100% rename from files/migrations/1698135711_create_logs_table.up.sql rename to files/migrations/1_init.sql diff --git a/files/vector.yaml b/files/vector.yaml index da33765..808e055 100644 --- a/files/vector.yaml +++ b/files/vector.yaml @@ -40,4 +40,4 @@ sinks: user: default password: default database: default - table: log_docker_raw + table: log_docker_raw \ No newline at end of file