모니터링 인프라 구축: ELK + Prometheus/Grafana + Slack 알림

.env.example

@@ -0,0 +1,4 @@
+# 로컬 모니터링 스택 실행 전, 이 파일을 .env로 복사하고 실제 값을 채우세요.
+# .env는 .gitignore에 포함되어 있어 커밋되지 않습니다.
+
+SLACK_WEBHOOK_URL=https://hooks.slack.com/services/XXXXXXXX/XXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX

.github/CODEOWNERS

@@ -1 +0,0 @@
-*.java @funnysunny08 @RinRinPARK @YuSuhwa-ve

.github/workflows/dev-cd.yml

@@ -33,7 +33,7 @@ jobs:
     - name: make application.properties 파일 생성
       run: |
         ## create application.yml
-        mkdir ./src/main/resources
+        mkdir -p ./src/main/resources
         cd ./src/main/resources
 
         # application.yml 파일 생성

.github/workflows/dev-ci.yml

@@ -30,7 +30,7 @@ jobs:
     - name: make application.properties 파일 생성
       run: |
         ## create application.yml
-        mkdir ./src/main/resources
+        mkdir -p ./src/main/resources
         cd ./src/main/resources
 
         # application.yml 파일 생성

.github/workflows/prod-cd.yml

@@ -34,7 +34,7 @@ jobs:
     - name: make application.properties 파일 생성
       run: |
         ## create application.yml
-        mkdir ./src/main/resources
+        mkdir -p ./src/main/resources
         cd ./src/main/resources
 
         # application.yml 파일 생성

.github/workflows/prod-ci.yml

@@ -23,7 +23,7 @@ jobs:
       - name: make application.properties 파일 생성
         run: |
           ## create application.yml
-          mkdir ./src/main/resources
+          mkdir -p ./src/main/resources
           cd ./src/main/resources
 
           # application.yml 파일 생성

.gitignore

@@ -42,3 +42,6 @@ application.yaml
 
 ### DS_Store
 .DS_Store
+logs/
+test_write.tmp
+.env

Dockerfile

@@ -0,0 +1,26 @@
+# --- Build stage ---
+FROM gradle:8.2.1-jdk11 AS build
+WORKDIR /app
+
+COPY build.gradle settings.gradle ./
+COPY gradle ./gradle
+COPY gradlew ./
+
+COPY src ./src
+
+# 로컬/모니터링 검증용 application.yml은 build context 루트에 두고 복사한다.
+# (CI/CD에서는 GitHub Secret으로 별도 생성 — 이 Dockerfile은 로컬 검증 전용)
+COPY application.yml ./application.yml
+
+RUN ./gradlew bootJar -x test --no-daemon
+
+# --- Run stage ---
+FROM eclipse-temurin:11-jre
+WORKDIR /app
+
+COPY --from=build /app/build/libs/*.jar app.jar
+COPY --from=build /app/application.yml application.yml
+
+EXPOSE 8080
+
+ENTRYPOINT ["java", "-Duser.timezone=Asia/Seoul", "-jar", "app.jar", "--spring.config.location=file:./application.yml"]

build.gradle

@@ -31,6 +31,9 @@ dependencies {
 	// Health Check
 	implementation 'org.springframework.boot:spring-boot-starter-actuator'
 
+	// Monitoring - Prometheus
+	implementation 'io.micrometer:micrometer-registry-prometheus:1.9.14'
+
 	// JPA & Database
 	implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
 	implementation group: 'org.postgresql', name: 'postgresql', version: '42.2.27'

docker-compose.yml

@@ -0,0 +1,115 @@
+version: "3.8"
+
+services:
+  runnect-app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: runnect-app
+    ports:
+      - "8080:8080"
+    depends_on:
+      - postgres
+      - redis
+    volumes:
+      - ./logs:/app/logs
+    networks:
+      - runnect-net
+
+  postgres:
+    image: postgres:15
+    container_name: runnect-postgres
+    environment:
+      POSTGRES_DB: runnect
+      POSTGRES_USER: runnect
+      POSTGRES_PASSWORD: runnect_local_password
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    networks:
+      - runnect-net
+
+  redis:
+    image: redis:7
+    container_name: runnect-redis
+    ports:
+      - "6379:6379"
+    networks:
+      - runnect-net
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4
+    container_name: runnect-elasticsearch
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ports:
+      - "9200:9200"
+    networks:
+      - runnect-net
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:8.13.4
+    container_name: runnect-logstash
+    depends_on:
+      - elasticsearch
+    volumes:
+      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
+      - ./logs:/usr/share/logstash/logs
+      - logstash_data:/usr/share/logstash/data
+    ports:
+      - "5044:5044"
+    networks:
+      - runnect-net
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.13.4
+    container_name: runnect-kibana
+    depends_on:
+      - elasticsearch
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+    ports:
+      - "5601:5601"
+    networks:
+      - runnect-net
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: runnect-prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    depends_on:
+      - runnect-app
+    networks:
+      - runnect-net
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: runnect-grafana
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - grafana_data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    depends_on:
+      - prometheus
+    networks:
+      - runnect-net
+
+networks:
+  runnect-net:
+    driver: bridge
+
+volumes:
+  postgres_data:
+  grafana_data:
+  logstash_data:

grafana/provisioning/alerting/contact-points.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: slack-alert
+    receivers:
+      - uid: slack-alert-receiver
+        type: slack
+        settings:
+          url: $__env{SLACK_WEBHOOK_URL}
+          username: "Grafana"

grafana/provisioning/alerting/contact-points.yml.tpl

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: slack-alert
+    receivers:
+      - uid: slack-alert-receiver
+        type: slack
+        settings:
+          url: "__SLACK_WEBHOOK_URL__"
+          username: "Grafana"

grafana/provisioning/alerting/notification-policies.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: slack-alert
+    group_by: ["alertname"]
+    group_wait: 10s
+    group_interval: 1m
+    repeat_interval: 10m

grafana/provisioning/alerting/rules.yml

@@ -0,0 +1,48 @@
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: runnect-alerts
+    folder: Runnect Alerts
+    interval: 30s
+    rules:
+      - uid: runnect-server-down
+        title: "Runnect Server Down"
+        condition: C
+        for: 30s
+        noDataState: Alerting
+        execErrState: Alerting
+        data:
+          - refId: A
+            datasourceUid: prometheus-ds
+            relativeTimeRange:
+              from: 120
+              to: 0
+            model:
+              expr: 'up{job="runnect-server"}'
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: "-100"
+            relativeTimeRange:
+              from: 120
+              to: 0
+            model:
+              type: threshold
+              refId: C
+              expression: A
+              conditions:
+                - evaluator:
+                    type: lt
+                    params: [1]
+                  operator:
+                    type: and
+                  query:
+                    params: ["A"]
+                  reducer:
+                    type: last
+                    params: []
+        annotations:
+          summary: "Runnect 서버가 응답하지 않고 있습니다 (up == 0)"
+        labels:
+          severity: critical

grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: "Runnect Dashboards"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    options:
+      path: /etc/grafana/provisioning/dashboards/json
+      foldersFromFilesStructure: true

grafana/provisioning/dashboards/json/runnect-overview.json

@@ -0,0 +1,56 @@
+{
+  "title": "Runnect Server Overview",
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "10s",
+  "panels": [
+    {
+      "id": 1,
+      "title": "JVM Heap Used (bytes)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "targets": [
+        {
+          "expr": "sum(jvm_memory_used_bytes{area=\"heap\", application=\"runnect-server\"})",
+          "legendFormat": "heap used"
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "CPU Usage",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "targets": [
+        {
+          "expr": "process_cpu_usage{application=\"runnect-server\"}",
+          "legendFormat": "cpu usage"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "HTTP Requests per Second",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "targets": [
+        {
+          "expr": "sum(rate(http_server_requests_seconds_count{application=\"runnect-server\"}[1m])) by (status)",
+          "legendFormat": "status {{status}}"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Server Up",
+      "type": "stat",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "targets": [
+        {
+          "expr": "up{job=\"runnect-server\"}",
+          "legendFormat": "up"
+        }
+      ]
+    }
+  ]
+}

grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: prometheus-ds
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

logstash.conf

@@ -0,0 +1,32 @@
+input {
+  file {
+    path => "/usr/share/logstash/logs/runnect-server.log"
+    start_position => "beginning"
+    codec => multiline {
+      pattern => "^%{TIMESTAMP_ISO8601}"
+      negate => true
+      what => "previous"
+    }
+  }
+}
+
+filter {
+  grok {
+    match => {
+      "message" => "%{TIMESTAMP_ISO8601:log_timestamp} \[%{DATA:traceId}\] %{LOGLEVEL:level}%{SPACE}\[%{DATA:thread}\] %{DATA:logger} - %{GREEDYDATA:log_message}"
+    }
+  }
+
+  date {
+    match => ["log_timestamp", "yyyy-MM-dd HH:mm:ss.SSS"]
+    timezone => "Asia/Seoul"
+    target => "@timestamp"
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["http://elasticsearch:9200"]
+    index => "runnect-logs-%{+yyyy.MM.dd}"
+  }
+}