harmony-one · sophoah · Sep 24, 2024 · Sep 16, 2024 · Sep 20, 2024
diff --git a/.gitignore b/.gitignore
@@ -98,3 +98,6 @@ profiles/*.pb.gz
 # cache db
 cache/
 cache_*_db
+
+# log aggregation logs folder variable file, no sense to track it
+test/logs_aggregator/.env
diff --git a/Makefile b/Makefile
@@ -54,6 +54,9 @@ help:
 	@echo "travis_rosetta_checker - run the Travis Rosetta checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH"
 	@echo "debug_external - cleans up environment, rebuilds the binary, and deploys with external nodes"
 	@echo "build_localnet_validator - imports validator keys, funds validator accounts, waits for the epoch, and creates external validators on a local network"
+	@echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, needs docker compose and started localnet"
+	@echo "debug-stop-log - stops a docker compose Promtail->Loki->Grafana stack"
+	@echo "debug-restart-log - restart a docker compose Promtail->Loki->Grafana stack"
 
 libs:
 	make -C $(TOP)/mcl -j8
@@ -216,3 +219,11 @@ build_localnet_validator:
 
 protofiles:
 	bash ./scripts/gogenerate.sh
+
+debug-start-log:
+	bash ./test/logs_aggregator/start_log_aggregator.sh
+
+debug-stop-log:
+	bash ./test/logs_aggregator/stop_log_aggregator.sh
+
+debug-restart-log: debug-stop-log debug-start-log
diff --git a/scripts/docker/README.md b/scripts/docker/README.md
@@ -34,7 +34,7 @@ If you need to open another shell, just do:
 docker exec -it harmonyone/harmony /bin/bash
 ```
 
-We also provide a `docker-compose` file for local testing
+We also provide a `docker compose` file for local testing
 
 To use the container in kubernetes, you can use a configmap or secret to mount the `harmony.conf` into the container
 ```bash
@@ -67,4 +67,4 @@ metadata:
 data:
   harmony.conf: |
     ...
-```
+```
diff --git a/test/logs_aggregator/docker-compose.yml b/test/logs_aggregator/docker-compose.yml
@@ -0,0 +1,25 @@
+services:
+  loki:
+    container_name: loki
+    image: grafana/loki:latest
+    ports:
+      - 3100:3100
+    volumes:
+      - ./loki-config.yml:/etc/loki/loki-config.yaml
+  promtail:
+    container_name: promtail
+    image: grafana/promtail:latest
+    volumes:
+      - ./promtail-config.yml:/etc/promtail/promtail-config.yaml
+      - ${CURRENT_SESSION_LOGS}:/var/log/
+  grafana:
+    container_name: grafana
+    image: grafana/grafana
+    # DANGER: USE THIS TWO OPTIONS ONLY AGAINST LOCAL deploy
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    ports:
+      - 3000:3000
+    volumes:
+      - ./loki-datasource.yaml:/etc/grafana/provisioning/datasources/loki-datasource.yaml
diff --git a/test/logs_aggregator/loki-config.yml b/test/logs_aggregator/loki-config.yml
@@ -0,0 +1,34 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 10096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
diff --git a/test/logs_aggregator/loki-datasource.yaml b/test/logs_aggregator/loki-datasource.yaml
@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    jsonData:
+      maxLines: 1000
diff --git a/test/logs_aggregator/promtail-config.yml b/test/logs_aggregator/promtail-config.yml
@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 10080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - "localhost"
+    labels:
+      job: varlogs
+      __path__: /var/log/*.log
diff --git a/test/logs_aggregator/start_log_aggregator.sh b/test/logs_aggregator/start_log_aggregator.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+echo "working in ${DIR}"
+cd $DIR && pwd
+logs="$(ls -1 '../../tmp_log/')"
+path="$(readlink -f '../../tmp_log/')"
+echo "Current localnet logs are placed into '${path}/${logs}'"
+echo "CURRENT_SESSION_LOGS='${path}/${logs}'" > .env
+echo "CURRENT_SESSION_LOGS='${path}/${logs}'"
+echo "starting docker compose lor log aggregation"
+docker compose up --detach
+sleep 5
+echo "Opening Grafana"
+python3 -m webbrowser "http://localhost:3000/explore"
diff --git a/test/logs_aggregator/stop_log_aggregator.sh b/test/logs_aggregator/stop_log_aggregator.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+echo "working in ${DIR}"
+cd $DIR && pwd
+echo "[INFO] - stopping log aggregation"
+docker compose rm --force --stop --volumes
+echo "[INFO] - cleanup .env"
+echo > .env