update

dashboards/Logs + Structured via Query.json

@@ -18,7 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 3,
+  "id": 1,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -38,7 +38,7 @@
     {
       "datasource": {
         "type": "grafana-clickhouse-datasource",
-        "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+        "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
       },
       "description": "",
       "fieldConfig": {
@@ -47,6 +47,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "series",
             "axisLabel": "",
@@ -174,7 +175,7 @@
           },
           "datasource": {
             "type": "grafana-clickhouse-datasource",
-            "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+            "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
           },
           "queryType": "builder",
           "rawSql": "SELECT $__timeInterval(logdatetime) as time,  count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND   ( logdatetime  >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
@@ -187,7 +188,7 @@
     {
       "datasource": {
         "type": "grafana-clickhouse-datasource",
-        "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+        "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
       },
       "fieldConfig": {
         "defaults": {
@@ -195,6 +196,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -311,7 +313,7 @@
           },
           "datasource": {
             "type": "grafana-clickhouse-datasource",
-            "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+            "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
           },
           "queryType": "builder",
           "rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND   ( logdatetime  >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
@@ -337,7 +339,7 @@
     {
       "datasource": {
         "type": "grafana-clickhouse-datasource",
-        "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+        "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
       },
       "fieldConfig": {
         "defaults": {
@@ -622,7 +624,7 @@
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "10.1.5",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "builderOptions": {
@@ -674,7 +676,7 @@
           },
           "datasource": {
             "type": "grafana-clickhouse-datasource",
-            "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+            "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
           },
           "format": 1,
           "meta": {
@@ -727,7 +729,7 @@
             }
           },
           "queryType": "sql",
-          "rawSql": "SELECT \n  visitParamExtractString(message, 'time') as time,\n  visitParamExtractString(message, 'level') as level,\n  visitParamExtractString(message, 'connection') as connection,\n  visitParamExtractString(message, 'account') as account,\n  visitParamExtractString(message, 'handler') as handler,\n  visitParamExtractString(message, 'msg') as msg,\n  mapFilter((k, v) -> k NOT IN ('time', 'level', 'connection', 'account', 'handler', 'msg'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE   ( logdatetime  >= $__fromTime AND logdatetime <= $__toTime )\n  AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n  AND ( program = ${program:singlequote} )\n  AND ( message LIKE '%$filter1%' )\n  AND ( message LIKE '%$filter2%' )\n  AND ( message LIKE '%$filter3%' )\n  AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000",
+          "rawSql": "SELECT \n  visitParamExtractString(message, 'datetime') as time,\n  visitParamExtractString(message, 'level_name') as level,\n  visitParamExtractString(message, 'connection') as connection,\n  visitParamExtractString(message, 'account') as account,\n  visitParamExtractString(message, 'handler') as handler,\n  visitParamExtractString(message, 'message') as msg,\n  mapFilter((k, v) -> k NOT IN ('datetime', 'level_name', 'connection', 'account', 'handler', 'message'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE   ( logdatetime  >= $__fromTime AND logdatetime <= $__toTime )\n  AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n  AND ( program = ${program:singlequote} )\n  AND ( message LIKE '%$filter1%' )\n  AND ( message LIKE '%$filter2%' )\n  AND ( message LIKE '%$filter3%' )\n  AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000",
           "refId": "A",
           "selectedFormat": 4
         }
@@ -754,7 +756,7 @@
       "type": "table"
     },
     {
-      "collapsed": true,
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -762,11 +764,14 @@
         "y": 37
       },
       "id": 1,
-      "panels": [
+      "panels": [],
+      "title": "Logs",
+      "type": "row"
+    },
     {
       "datasource": {
         "type": "grafana-clickhouse-datasource",
-            "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+        "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
       },
       "fieldConfig": {
         "defaults": {
@@ -867,7 +872,7 @@
         "h": 27,
         "w": 24,
         "x": 0,
-            "y": 11
+        "y": 38
       },
       "id": 2,
       "options": {
@@ -883,7 +888,7 @@
         "showHeader": true,
         "sortBy": []
       },
-          "pluginVersion": "10.1.5",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "builderOptions": {
@@ -936,7 +941,7 @@
           },
           "datasource": {
             "type": "grafana-clickhouse-datasource",
-                "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+            "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
           },
           "format": 1,
           "meta": {
@@ -956,13 +961,8 @@
       "type": "table"
     }
   ],
-      "title": "Logs",
-      "type": "row"
-    }
-  ],
   "refresh": "",
-  "schemaVersion": 38,
+  "schemaVersion": 39,
-  "style": "dark",
   "tags": [],
   "templating": {
     "list": [
@@ -974,7 +974,7 @@
         },
         "datasource": {
           "type": "grafana-clickhouse-datasource",
-          "uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
+          "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
         },
         "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;",
         "hide": 0,
@@ -1085,6 +1085,6 @@
   "timezone": "",
   "title": "Logs + Structured via Query",
   "uid": "b445eb43-25d1-4112-b628-3b8acf037f10",
-  "version": 8,
+  "version": 4,
   "weekStart": ""
 }

docker-compose.yml

@@ -22,7 +22,7 @@ services:
       - '8686:8686'
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
-      - ./files/vector.toml:/etc/vector/vector.toml:ro
+      - ./files/vector.yaml:/etc/vector/vector.yaml:ro
 
 volumes:
   clickhouse_data:

files/vector.toml

@@ -1,38 +0,0 @@
-[api]
-enabled = true
-address = "0.0.0.0:8686"
-
-[sources.docker]
-type = "docker_logs"
-exclude_containers = [ "clickhouse", "grafana", "vector" ]
-
-[transforms.ts_and_program]
-type = "remap"
-inputs = [ "docker" ]
-source = '''
-  .logdatetime = to_unix_timestamp(now())
-  .program = .container_name
-'''
-
-[transforms.emptyprogram]
-type = "filter"
-inputs = [ "ts_and_program" ]
-condition.type = "vrl"
-condition.source = '''
-  .program != ""
-'''
-
-[transforms.dedupe]
-type = "dedupe"
-inputs = [ "emptyprogram" ]
-
-[sinks.clickhouse]
-type = "clickhouse"
-inputs = [ "dedupe"] 
-compression = "gzip"
-endpoint = "http://clickhouse:8123"
-auth.strategy = "basic"
-auth.user = "default"
-auth.password = "default"
-database = "default"
-table = "log_docker_raw"

files/vector.yaml

@@ -0,0 +1,43 @@
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  docker:
+    type: docker_logs
+    exclude_containers:
+      - clickhouse
+      - grafana
+      - vector
+transforms:
+  ts_and_program:
+    type: remap
+    inputs:
+      - docker
+    source: |2
+        .logdatetime = to_unix_timestamp(now())
+        .program = .container_name
+  emptyprogram:
+    type: filter
+    inputs:
+      - ts_and_program
+    condition:
+      type: vrl
+      source: |2
+          .program != ""
+  dedupe:
+    type: dedupe
+    inputs:
+      - emptyprogram
+sinks:
+  clickhouse:
+    type: clickhouse
+    inputs:
+      - dedupe
+    compression: gzip
+    endpoint: http://clickhouse:8123
+    auth:
+      strategy: basic
+      user: default
+      password: default
+    database: default
+    table: log_docker_raw