From 9c4e22efa189fec6671745465e2eb1aa535d15bf Mon Sep 17 00:00:00 2001 From: Hexeong <123macanic@naver.com> Date: Thu, 20 Aug 2026 20:57:24 +0900 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20DB=20=EB=B0=B1=EC=97=85=20=EC=8B=A4?= =?UTF-8?q?=ED=8C=A8=20=EC=95=8C=EB=A6=BC=20API=20=EC=97=B0=EB=8F=99=20?= =?UTF-8?q?=EC=9E=91=EC=97=85=20=EA=B5=AC=ED=98=84=20-=20systemd=EB=A1=9C?= =?UTF-8?q?=20=EC=8B=A4=ED=96=89=EB=90=98=EB=8A=94=20=EB=B0=B1=EC=97=85=20?= =?UTF-8?q?=EC=8A=A4=EC=BC=80=EC=A5=B4=EC=97=90=20=EC=8B=A4=ED=8C=A8?= =?UTF-8?q?=EC=8B=9C=20discord=20API=EB=A1=9C=20=EC=9A=94=EC=B2=AD?= =?UTF-8?q?=EC=9D=84=20=EB=B3=B4=EB=82=B4=EB=8F=84=EB=A1=9D=20=EC=88=98?= =?UTF-8?q?=EC=A0=95=20-=20aws=20cli=20=EB=93=B1=20db=5Fec2=EC=97=90=20?= =?UTF-8?q?=ED=95=84=EC=9A=94=ED=95=9C=20=EB=8F=84=EA=B5=AC=EB=A5=BC=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80=ED=95=9C=20=EC=83=88=EB=A1=9C=EC=9A=B4=20ami?= =?UTF-8?q?=20=EB=B0=98=EC=98=81=20-=20api=5Fserver=EC=97=90=EC=84=9C=20db?= =?UTF-8?q?=5Fec2=EC=97=90=EC=84=9C=20=EB=B3=B4=EB=82=B4=EB=8A=94=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=EC=9D=84=20=EB=B0=9B=EB=8F=84=EB=A1=9D=20?= =?UTF-8?q?=EC=9D=B8=EB=B0=94=EC=9A=B4=EB=93=9C=20=EA=B7=9C=EC=B9=99=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/mysql-backup-deploy.yml | 44 +++- config/secrets | 2 +- environment/prod/main.tf | 11 +- environment/prod/variables.tf | 11 + environment/stage/main.tf | 4 + modules/app_stack/security_groups.tf | 14 ++ modules/app_stack/variables.tf | 5 + scripts/mysql_backup/README.md | 37 +++- scripts/mysql_backup/bin/mysql-backup-binlog | 43 ++-- scripts/mysql_backup/bin/mysql-backup-dump | 36 ++- .../mysql_backup/bin/mysql-backup-validate | 18 +- scripts/mysql_backup/install.sh | 4 +- scripts/mysql_backup/lib/backup-common.sh | 133 +++++++++++ scripts/mysql_backup/tests/run.sh | 206 ++++++++++++++++++ scripts/mysql_backup/validate-remote.sh | 32 ++- 15 files changed, 549 insertions(+), 51 deletions(-) diff --git a/.github/workflows/mysql-backup-deploy.yml b/.github/workflows/mysql-backup-deploy.yml index 3c12049..8567682 100644 --- a/.github/workflows/mysql-backup-deploy.yml +++ b/.github/workflows/mysql-backup-deploy.yml @@ -34,6 +34,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false + submodules: recursive + token: ${{ secrets.GH_PAT }} - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: @@ -92,16 +94,29 @@ jobs: - name: Validate or install MySQL backup env: DEPLOY_MODE: ${{ inputs.mode }} - MYSQL_BACKUP_BUCKET: ${{ vars.MYSQL_BACKUP_BUCKET_NAME }} - MYSQL_DATABASE: ${{ vars.MYSQL_BACKUP_DATABASE_NAME }} - DB_HOST_FINGERPRINT: ${{ vars.PROD_DB_SSH_HOST_KEY_ED25519 }} + MYSQL_BACKUP_BUCKET: ${{ secrets.MYSQL_BACKUP_BUCKET_NAME }} + MYSQL_DATABASE: ${{ secrets.MYSQL_BACKUP_DATABASE_NAME }} + DB_HOST_FINGERPRINT: ${{ secrets.PROD_DB_SSH_HOST_KEY_ED25519 }} + ALARM_TFVARS_PATH: config/secrets/prod_db.tfvars run: | set -Eeuo pipefail umask 077 - : "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET_NAME repository variable is required}" - : "${MYSQL_DATABASE:?MYSQL_BACKUP_DATABASE_NAME repository variable is required}" - : "${DB_HOST_FINGERPRINT:?PROD_DB_SSH_HOST_KEY_ED25519 repository variable is required}" + : "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET_NAME repository secret is required}" + : "${MYSQL_DATABASE:?MYSQL_BACKUP_DATABASE_NAME repository secret is required}" + : "${DB_HOST_FINGERPRINT:?PROD_DB_SSH_HOST_KEY_ED25519 repository secret is required}" + + # 알림 토큰은 tfvars 를 단일 원천으로 두므로 secrets submodule 에서 읽는다. + if [[ ! -f "$ALARM_TFVARS_PATH" ]]; then + echo "::error::$ALARM_TFVARS_PATH is missing; check the secrets submodule checkout" + exit 1 + fi + ALARM_API_TOKEN="$(sed -n 's/^[[:space:]]*mysql_backup_fail_alarm_request_token[[:space:]]*=[[:space:]]*"\(.*\)"[[:space:]]*$/\1/p' "$ALARM_TFVARS_PATH" | head -1)" + if [[ -z "$ALARM_API_TOKEN" ]]; then + echo "::error::mysql_backup_fail_alarm_request_token is missing in $ALARM_TFVARS_PATH" + exit 1 + fi + echo "::add-mask::$ALARM_API_TOKEN" if [[ "$DEPLOY_MODE" != "validate" && "$DEPLOY_MODE" != "install" ]]; then echo "::error::Invalid deployment mode" exit 1 @@ -162,6 +177,18 @@ jobs: exit 1 fi + # DB EC2 는 인터넷 경로가 없어 API EC2 의 private ip 로 알림을 보낸다. + ALARM_API_HOST="$(aws ec2 describe-instances \ + --instance-ids "$API_INSTANCE_ID" \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' \ + --output text)" + if [[ -z "$ALARM_API_HOST" || "$ALARM_API_HOST" == "None" ]]; then + echo "::error::Prod API EC2 private IP was not found" + exit 1 + fi + # Blue/Green 활성 슬롯을 알 수 없으므로 두 슬롯의 app 포트를 순서대로 시도한다. + ALARM_API_PORTS="8080 9080" + aws ssm start-session \ --target "$API_INSTANCE_ID" \ --document-name AWS-StartPortForwardingSessionToRemoteHost \ @@ -202,7 +229,7 @@ jobs: --instance-id "$DB_INSTANCE_ID" \ --instance-os-user ubuntu \ --ssh-public-key "file://$KEY_DIR/id_ed25519.pub" >/dev/null - REMOTE_VALIDATE_COMMAND="env MYSQL_BACKUP_BUCKET=$(printf '%q' "$MYSQL_BACKUP_BUCKET") MYSQL_DATABASE=$(printf '%q' "$MYSQL_DATABASE") AWS_REGION=$(printf '%q' "$AWS_REGION") bash -s" + REMOTE_VALIDATE_COMMAND="env MYSQL_BACKUP_BUCKET=$(printf '%q' "$MYSQL_BACKUP_BUCKET") MYSQL_DATABASE=$(printf '%q' "$MYSQL_DATABASE") AWS_REGION=$(printf '%q' "$AWS_REGION") ALARM_API_HOST=$(printf '%q' "$ALARM_API_HOST") ALARM_API_PORTS=$(printf '%q' "$ALARM_API_PORTS") ALARM_API_TOKEN=$(printf '%q' "$ALARM_API_TOKEN") bash -s" ssh "${SSH_OPTIONS[@]}" ubuntu@127.0.0.1 \ "sudo bash -c $(printf '%q' "$REMOTE_VALIDATE_COMMAND")" \ < scripts/mysql_backup/validate-remote.sh @@ -214,6 +241,9 @@ jobs: printf 'MYSQL_BACKUP_BUCKET=%s\n' "$MYSQL_BACKUP_BUCKET" printf 'MYSQL_DATABASE=%s\n' "$MYSQL_DATABASE" printf 'AWS_REGION=%s\n' "$AWS_REGION" + printf 'ALARM_API_HOST=%s\n' "$ALARM_API_HOST" + printf 'ALARM_API_PORTS=%s\n' "$ALARM_API_PORTS" + printf 'ALARM_API_TOKEN=%s\n' "$ALARM_API_TOKEN" } >"$CONFIG_FILE" cp -R scripts/mysql_backup "$KEY_DIR/bundle" diff --git a/config/secrets b/config/secrets index c9f90e3..af241b3 160000 --- a/config/secrets +++ b/config/secrets @@ -1 +1 @@ -Subproject commit c9f90e38261e50b7ffab749a37afc25dade1721a +Subproject commit af241b33d504e8b63a79768ce2d14c580159be25 diff --git a/environment/prod/main.tf b/environment/prod/main.tf index ab7a423..3b472ef 100644 --- a/environment/prod/main.tf +++ b/environment/prod/main.tf @@ -22,11 +22,12 @@ module "prod_stack" { db_instance_class = var.db_instance_class # DB EC2 설정 - enable_db_ec2 = true - db_instance_type = var.db_ec2_instance_type - db_ami_id = var.db_ec2_ami_id - db_subnet_id = var.db_ec2_subnet_id - db_data_volume_size = var.db_data_volume_size + enable_db_ec2 = true + internal_alarm_api_ports = var.internal_alarm_api_ports + db_instance_type = var.db_ec2_instance_type + db_ami_id = var.db_ec2_ami_id + db_subnet_id = var.db_ec2_subnet_id + db_data_volume_size = var.db_data_volume_size # 보안 그룹 규칙 api_ingress_rules = var.api_ingress_rules diff --git a/environment/prod/variables.tf b/environment/prod/variables.tf index 9236616..76210d6 100644 --- a/environment/prod/variables.tf +++ b/environment/prod/variables.tf @@ -159,3 +159,14 @@ variable "alloy_version" { description = "Docker image tag for Grafana Alloy" type = string } + +variable "mysql_backup_fail_alarm_request_token" { + description = "백업 실패 알림 API 호출에 사용하는 공유 토큰. Terraform은 이 값을 사용하지 않고 배포 워크플로우가 tfvars에서 직접 읽는다." + type = string + sensitive = true +} + +variable "internal_alarm_api_ports" { + description = "DB EC2가 백업 실패 알림을 보내는 API 서버의 Blue/Green app 포트" + type = list(number) +} diff --git a/environment/stage/main.tf b/environment/stage/main.tf index 4921402..ab0342f 100644 --- a/environment/stage/main.tf +++ b/environment/stage/main.tf @@ -6,6 +6,10 @@ data "aws_vpc" "default" { module "stage_stack" { source = "../../modules/app_stack" + # stage 는 DB 가 API 인스턴스의 컨테이너로 떠 있어 별도 DB EC2 가 없다. + # enable_db_ec2 가 false 라 알림 인그레스가 생성되지 않으므로 빈 목록을 넘긴다. + internal_alarm_api_ports = [] + env_name = "stage" vpc_id = data.aws_vpc.default.id diff --git a/modules/app_stack/security_groups.tf b/modules/app_stack/security_groups.tf index 0de5d32..50be812 100644 --- a/modules/app_stack/security_groups.tf +++ b/modules/app_stack/security_groups.tf @@ -15,6 +15,20 @@ resource "aws_security_group" "api_sg" { } } + # DB EC2 는 인터넷 경로가 없어 API EC2 의 app 포트로 백업 실패 알림을 보냅니다. + # Blue/Green 활성 슬롯을 알 수 없어 두 슬롯의 포트를 모두 열고, 소스는 DB EC2 서브넷으로 제한합니다. + # db_ec2_sg 가 이미 api_sg 를 참조하므로 보안 그룹을 소스로 쓰면 순환 참조가 되어 서브넷 CIDR 을 사용합니다. + dynamic "ingress" { + for_each = var.enable_db_ec2 ? toset(var.internal_alarm_api_ports) : toset([]) + content { + description = "Internal backup alarm from DB EC2 subnet" + from_port = ingress.value + to_port = ingress.value + protocol = "tcp" + cidr_blocks = [data.aws_subnet.db_ec2[0].cidr_block] + } + } + # [Outbound] 모든 트래픽 허용 egress { from_port = 0 diff --git a/modules/app_stack/variables.tf b/modules/app_stack/variables.tf index c4bb113..7f511fa 100644 --- a/modules/app_stack/variables.tf +++ b/modules/app_stack/variables.tf @@ -201,3 +201,8 @@ variable "alloy_version" { description = "Docker image tag for Grafana Alloy" type = string } + +variable "internal_alarm_api_ports" { + description = "DB EC2가 백업 실패 알림을 보내는 API 서버의 Blue/Green app 포트" + type = list(number) +} diff --git a/scripts/mysql_backup/README.md b/scripts/mysql_backup/README.md index 91a999a..b08a7b7 100644 --- a/scripts/mysql_backup/README.md +++ b/scripts/mysql_backup/README.md @@ -12,9 +12,7 @@ DB EC2에서 다음 systemd 작업을 실행합니다. Repository Secrets: - `AWS_ROLE_ARN`: 배포 워크플로우가 AssumeRole할 IAM 역할 ARN - -Repository Variables: - +- `GH_PAT`: secrets submodule을 체크아웃할 토큰 - `MYSQL_BACKUP_BUCKET_NAME`: 백업 버킷 이름 - `MYSQL_BACKUP_DATABASE_NAME`: 백업할 DB 이름. 필수값이며 공개 코드에 기본값을 두지 않습니다. - `PROD_DB_SSH_HOST_KEY_ED25519`: DB EC2의 ED25519 host key SHA-256 fingerprint @@ -27,6 +25,39 @@ GitHub Environment: `MySQL Backup Test` 워크플로우는 AWS 권한이나 운영 환경 접근 없이 백업 스크립트 단위 테스트를 수동으로 실행합니다. +## 백업 실패 알림 + +백업이 실패하거나 지연되면 API 서버의 내부 전용 API를 거쳐 Discord로 알립니다. + +```text +DB EC2 (private subnet, 인터넷 경로 없음) +└─ 백업 실패 감지 + └─ POST http://:<8080 또는 9080>/internal/alarms/db-backup + └─ API 서버 → Discord Webhook +``` + +- DB EC2가 있는 서브넷의 라우팅 테이블에는 NAT와 IGW가 없어 Discord를 직접 호출할 수 없으므로 API 서버가 중계합니다. +- Blue/Green 활성 슬롯을 알 수 없으므로 두 슬롯의 app 포트를 순서대로 시도하고 먼저 응답한 쪽으로 보냅니다. +- API EC2 보안 그룹은 이 두 포트를 DB EC2 서브넷에서만 허용합니다. +- 알림 전송 실패는 백업 자체를 실패시키지 않고 로그로만 남깁니다. + +알림 유형: + +| 유형 | 발생 조건 | +|------|-----------| +| `DUMP_FAILED` | 여유 공간 부족, mysqldump 실패, 복구 기준점 누락, dump 업로드 실패 | +| `BINLOG_UPLOAD_FAILED` | binlog 회전 실패, binlog 업로드 실패 | +| `BINLOG_GAP_DETECTED` | binlog 번호 불연속, 번호 역행, 닫힌 파일 누락 | +| `BINLOG_UPLOAD_DELAYED` | 마지막 성공 업로드가 5분을 초과 | + +`BINLOG_UPLOAD_DELAYED`는 스크립트가 실행되고 있을 때만 감지할 수 있습니다. EC2나 타이머 자체가 멈춘 경우는 감지할 수 없으므로 S3의 마지막 객체 시각을 외부에서 관찰하는 모니터링이 별도로 필요합니다. + +### 알림 인증 토큰 + +호출자 인증 토큰은 `config/secrets/prod_db.tfvars`의 `mysql_backup_fail_alarm_request_token`을 단일 원천으로 사용합니다. 배포 워크플로우가 secrets submodule에서 값을 읽어 DB EC2의 `/etc/solid-connection/mysql-backup.env`에 기록하므로, 토큰을 변경할 때 Terraform apply 없이 배포 워크플로우만 다시 실행하면 됩니다. + +> API 서버는 같은 토큰을 Parameter Store의 `/solid-connection/{env}/internal-alarm.token`에서 읽습니다. 토큰을 회전할 때 두 곳을 함께 변경해야 하며, 한쪽만 변경하면 모든 알림이 401로 거부됩니다. + ## dump 실패 처리 - dump 실행 직전에 예상 dump 크기의 2배와 256MiB의 여유 공간을 확인합니다. diff --git a/scripts/mysql_backup/bin/mysql-backup-binlog b/scripts/mysql_backup/bin/mysql-backup-binlog index 0317003..5965a34 100644 --- a/scripts/mysql_backup/bin/mysql-backup-binlog +++ b/scripts/mysql_backup/bin/mysql-backup-binlog @@ -6,7 +6,15 @@ readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql- source "$LIB_DIR/backup-common.sh" require_backup_environment -require_commands aws docker flock sha256sum +require_commands aws curl docker flock sha256sum + +# 타이머 주기(5분)와 같은 값을 쓰면 정상 동작 중에도 경계에서 매번 지연으로 판정되므로 3주기로 둔다. +readonly UPLOAD_DELAY_THRESHOLD_SECONDS=900 + +# 명시적으로 처리하지 않은 실패도 알리도록 등록합니다. +trap 'alarm_on_unexpected_failure BINLOG_UPLOAD_FAILED' EXIT + +alarm_if_upload_delayed "$STATE_DIR/last-binlog-success" "$UPLOAD_DELAY_THRESHOLD_SECONDS" exec 9>"$STATE_DIR/binlog.lock" if ! flock -n 9; then @@ -53,8 +61,7 @@ host_binlog_index="$MYSQL_DATA_DIR/$(basename "$binlog_basename").index" if ! validate_binlog_name "$active_binlog" || \ [[ ! "$database_server_uuid" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then - echo "Unexpected MySQL binary log metadata." >&2 - exit 1 + fail_with_alarm BINLOG_UPLOAD_FAILED "unexpected mysql binary log metadata" fi if [[ ! -r "$host_binlog_index" ]]; then echo "MySQL binary log index is not readable: $host_binlog_index" >&2 @@ -131,8 +138,8 @@ if [[ -n "$last_uploaded" ]]; then active_number=$((10#${active_binlog##*.})) last_uploaded_number=$((10#${last_uploaded##*.})) if ((active_number <= last_uploaded_number)); then - echo "Binary log numbering moved backwards. Clear the state only after starting a new verified backup chain." >&2 - exit 1 + fail_with_alarm BINLOG_GAP_DETECTED \ + "binary log numbering moved backwards: active=$active_binlog last_uploaded=$last_uploaded" fi fi @@ -152,8 +159,7 @@ if [[ "$rotation_to" == "-" ]]; then observed_active_binlog="$(read_active_binlog)" fi if ! validate_binlog_name "$observed_active_binlog" || [[ "$observed_active_binlog" == "$rotation_from" ]]; then - echo "MySQL binary log rotation did not advance from $rotation_from." >&2 - exit 1 + fail_with_alarm BINLOG_UPLOAD_FAILED "binary log rotation did not advance from $rotation_from" fi rotation_to="$observed_active_binlog" write_binlog_state @@ -188,15 +194,14 @@ while IFS= read -r indexed_path; do previous_number=$((10#${previous_binlog##*.})) current_number=$((10#${binlog_name##*.})) if ((current_number != previous_number + 1)); then - echo "Binary log gap detected between $previous_binlog and $binlog_name." >&2 - exit 1 + fail_with_alarm BINLOG_GAP_DETECTED \ + "binary log gap detected between $previous_binlog and $binlog_name" fi fi binlog_file="$MYSQL_DATA_DIR/$binlog_name" if [[ ! -s "$binlog_file" ]]; then - echo "Closed binary log file is missing or empty: $binlog_file" >&2 - exit 1 + fail_with_alarm BINLOG_GAP_DETECTED "closed binary log file is missing or empty: $binlog_name" fi closed_epoch="$(stat -c %Y "$binlog_file")" @@ -220,9 +225,13 @@ while IFS= read -r indexed_path; do EOF object_prefix="binlog/$key_date/${key_time}-${database_server_uuid}-${binlog_name}" - upload_file_once "$binlog_file" "$object_prefix" + if ! upload_file_once "$binlog_file" "$object_prefix"; then + fail_with_alarm BINLOG_UPLOAD_FAILED "failed to upload the binary log to s3: $binlog_name" + fi # manifest가 존재하는 binlog만 복구 가능한 업로드 완료 파일로 취급합니다. - upload_file_once "$manifest_file" "$object_prefix.manifest.json" + if ! upload_file_once "$manifest_file" "$object_prefix.manifest.json"; then + fail_with_alarm BINLOG_UPLOAD_FAILED "failed to upload the binary log manifest to s3: $binlog_name" + fi rm -f "$manifest_file" last_uploaded="$binlog_name" @@ -232,15 +241,15 @@ EOF done <"$host_binlog_index" if [[ "$active_found" != "true" ]]; then - echo "Active binary log is not present in the binary log index: $active_binlog" >&2 - exit 1 + fail_with_alarm BINLOG_GAP_DETECTED \ + "active binary log is not present in the binary log index: $active_binlog" fi if [[ -n "$previous_binlog" ]]; then previous_number=$((10#${previous_binlog##*.})) active_number=$((10#${active_binlog##*.})) if ((active_number != previous_number + 1)); then - echo "Binary log gap detected between $previous_binlog and active log $active_binlog." >&2 - exit 1 + fail_with_alarm BINLOG_GAP_DETECTED \ + "binary log gap detected between $previous_binlog and active log $active_binlog" fi fi diff --git a/scripts/mysql_backup/bin/mysql-backup-dump b/scripts/mysql_backup/bin/mysql-backup-dump index 3fa57b6..f0f6931 100755 --- a/scripts/mysql_backup/bin/mysql-backup-dump +++ b/scripts/mysql_backup/bin/mysql-backup-dump @@ -6,10 +6,13 @@ readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql- source "$LIB_DIR/backup-common.sh" require_backup_environment -require_commands aws docker flock gzip sha256sum +require_commands aws curl docker flock gzip sha256sum readonly MAX_DUMP_JOB_AGE_SECONDS=21600 +# 명시적으로 처리하지 않은 실패도 알리도록 등록합니다. +trap 'alarm_on_unexpected_failure DUMP_FAILED' EXIT + exec 9>"$STATE_DIR/dump.lock" if ! flock -n 9; then echo "Another mysqldump backup is already running." >&2 @@ -64,15 +67,22 @@ install -d -m 700 "$JOB_DIR" if [[ ! -s "$DUMP_FILE" ]]; then partial_dump="$DUMP_FILE.partial" rm -f "$partial_dump" - space_status="$(require_dump_staging_space)" + # 성공 시 stdout 만 파싱해야 하므로 stderr 를 섞지 않고, 실패 원인은 스크립트 로그에 남긴다. + if ! space_status="$(require_dump_staging_space)"; then + fail_with_alarm DUMP_FAILED "insufficient staging space for the mysql dump" + fi read -r database_bytes available_bytes required_bytes <<<"$space_status" echo "MySQL dump staging space is sufficient: database_bytes=$database_bytes available_bytes=$available_bytes required_bytes=$required_bytes" - docker exec "$MYSQL_CONTAINER" sh -lc \ + if ! docker exec "$MYSQL_CONTAINER" sh -lc \ 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysqldump -uroot --single-transaction --quick --source-data=2 --routines --events --triggers --hex-blob --set-gtid-purged=OFF --no-tablespaces "$1"' \ - sh "$MYSQL_DATABASE" | gzip -1 >"$partial_dump" + sh "$MYSQL_DATABASE" | gzip -1 >"$partial_dump"; then + fail_with_alarm DUMP_FAILED "mysqldump failed for database $MYSQL_DATABASE" + fi - test -s "$partial_dump" + if [[ ! -s "$partial_dump" ]]; then + fail_with_alarm DUMP_FAILED "mysqldump produced an empty dump for database $MYSQL_DATABASE" + fi mv "$partial_dump" "$DUMP_FILE" fi @@ -81,8 +91,7 @@ printf '%s %s\n' "$dump_checksum" "$(basename "$DUMP_FILE")" >"$CHECKSUM_FILE" source_status="$(gzip -dc "$DUMP_FILE" | grep -m1 '^-- CHANGE REPLICATION SOURCE TO' || true)" if [[ -z "$source_status" ]]; then - echo "The dump does not contain a binary log recovery position." >&2 - exit 1 + fail_with_alarm DUMP_FAILED "the dump does not contain a binary log recovery position" fi source_file="$(sed -n "s/.*SOURCE_LOG_FILE='\([^']*\)'.*/\1/p" <<<"$source_status")" @@ -92,8 +101,7 @@ dump_size="$(stat -c %s "$DUMP_FILE")" created_at="${job_id:0:4}-${job_id:4:2}-${job_id:6:2}T${job_id:9:2}:${job_id:11:2}:${job_id:13:2}Z" if [[ ! "$source_file" =~ ^binlog\.[0-9]{6}$ || ! "$source_position" =~ ^[0-9]+$ || ! "$database_server_uuid" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then - echo "Failed to parse dump recovery metadata." >&2 - exit 1 + fail_with_alarm DUMP_FAILED "failed to parse dump recovery metadata" fi cat >"$MANIFEST_FILE" <"$MANIFEST_FILE" <"$STATE_DIR/last-dump-success.tmp" mv "$STATE_DIR/last-dump-success.tmp" "$STATE_DIR/last-dump-success" diff --git a/scripts/mysql_backup/bin/mysql-backup-validate b/scripts/mysql_backup/bin/mysql-backup-validate index a072981..29a3702 100755 --- a/scripts/mysql_backup/bin/mysql-backup-validate +++ b/scripts/mysql_backup/bin/mysql-backup-validate @@ -6,7 +6,8 @@ readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql- source "$LIB_DIR/backup-common.sh" require_backup_environment -require_commands aws docker flock gzip sha256sum +require_alarm_environment +require_commands aws curl docker flock gzip sha256sum timeout mountpoint -q /mnt/mysql-data docker inspect "$MYSQL_CONTAINER" >/dev/null @@ -42,5 +43,20 @@ aws s3api head-bucket \ --bucket "$MYSQL_BACKUP_BUCKET" \ --region "$AWS_REGION" >/dev/null +# 알림 경로가 실제로 닿는지 확인합니다. +# actuator 는 management 포트에만 열려 있고 비활성 슬롯은 내려가 있으므로, app 포트의 tcp 연결로 확인합니다. +alarm_endpoint_reachable=false +for alarm_port in ${ALARM_API_PORTS}; do + if timeout 3 bash -c "exec 3<>/dev/tcp/${ALARM_API_HOST}/${alarm_port}" 2>/dev/null; then + alarm_endpoint_reachable=true + echo "Alarm api is reachable on port $alarm_port." + break + fi +done +if [[ "$alarm_endpoint_reachable" != "true" ]]; then + echo "Alarm api is not reachable on any of the configured ports: $ALARM_API_PORTS" >&2 + exit 1 +fi + echo "MySQL backup validation succeeded." echo "database_bytes=$database_bytes available_bytes=$available_bytes required_bytes=$required_bytes" diff --git a/scripts/mysql_backup/install.sh b/scripts/mysql_backup/install.sh index db79395..12af6d1 100755 --- a/scripts/mysql_backup/install.sh +++ b/scripts/mysql_backup/install.sh @@ -23,7 +23,7 @@ if [[ ! -f "$CONFIG_SOURCE" ]]; then exit 1 fi -for command_name in aws bash cp docker flock gzip install mountpoint mv sha256sum systemctl systemd-analyze; do +for command_name in aws bash cp curl docker flock gzip install mountpoint mv sha256sum systemctl systemd-analyze; do command -v "$command_name" >/dev/null || { echo "Required command is not installed: $command_name" >&2 exit 1 @@ -66,7 +66,7 @@ load_candidate_config() { while IFS='=' read -r key value || [[ -n "$key" ]]; do [[ -z "$key" || "$key" == \#* ]] && continue case "$key" in - MYSQL_BACKUP_BUCKET|MYSQL_DATABASE|AWS_REGION) + MYSQL_BACKUP_BUCKET|MYSQL_DATABASE|AWS_REGION|ALARM_API_HOST|ALARM_API_PORTS|ALARM_API_TOKEN) printf -v "$key" '%s' "$value" export "$key" ;; diff --git a/scripts/mysql_backup/lib/backup-common.sh b/scripts/mysql_backup/lib/backup-common.sh index 077a131..c3cb0b1 100644 --- a/scripts/mysql_backup/lib/backup-common.sh +++ b/scripts/mysql_backup/lib/backup-common.sh @@ -7,6 +7,13 @@ readonly STATE_DIR="$BACKUP_ROOT/state" readonly MYSQL_DATA_DIR="${MYSQL_DATA_DIR:-/mnt/mysql-data/mysql}" readonly MYSQL_CONTAINER="${MYSQL_CONTAINER:-mysql-server}" readonly DUMP_SPACE_RESERVE_BYTES=268435456 +readonly ALARM_PATH="/internal/alarms/db-backup" +readonly ALARM_TIMEOUT_SECONDS=5 +readonly ALARM_RETRY_COUNT=2 +readonly ALARM_DETAIL_MAX_LENGTH=1000 + +# 같은 실패로 알림이 두 번 나가지 않도록 전송 여부를 기록합니다. +alarm_sent=false require_backup_environment() { : "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET is required}" @@ -130,3 +137,129 @@ upload_file_once() { --no-progress \ --metadata "sha256=$checksum" } + +require_alarm_environment() { + : "${ALARM_API_HOST:?ALARM_API_HOST is required}" + : "${ALARM_API_PORTS:?ALARM_API_PORTS is required}" + : "${ALARM_API_TOKEN:?ALARM_API_TOKEN is required}" + + if [[ ! "$ALARM_API_HOST" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "Invalid alarm api host." >&2 + return 1 + fi + if [[ ! "$ALARM_API_PORTS" =~ ^[0-9]+( [0-9]+)*$ ]]; then + echo "Invalid alarm api ports." >&2 + return 1 + fi +} + +instance_id() { + local metadata_token + + metadata_token="$(curl -fsS -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60" \ + --max-time 2 2>/dev/null)" || return 1 + curl -fsS -H "X-aws-ec2-metadata-token: $metadata_token" \ + "http://169.254.169.254/latest/meta-data/instance-id" \ + --max-time 2 2>/dev/null +} + +# sed 의 N 명령은 GNU 와 BSD 동작이 달라 한 줄 입력에서 결과가 사라지므로 bash 치환만 사용합니다. +json_escape() { + local value="$1" + + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="${value//$'\t'/ }" + value="${value//$'\n'/\\n}" + printf '%s' "$value" +} + +# 활성 슬롯을 알 수 없으므로 blue, green 순서로 시도하고 먼저 응답한 쪽으로 보냅니다. +# 알림 전송 실패가 백업 자체를 실패시키지 않도록 항상 0으로 종료합니다. +send_backup_alarm() { + local alarm_type="$1" + local detail="$2" + local target_instance_id + local header_config + local payload + local port + + if [[ -z "${ALARM_API_HOST:-}" || -z "${ALARM_API_TOKEN:-}" ]]; then + echo "Alarm target is not configured; skipping the backup alarm." >&2 + return 0 + fi + + target_instance_id="$(instance_id)" || target_instance_id="unknown" + payload="$(printf '{"type":"%s","instanceId":"%s","occurredAt":"%s","detail":"%s"}' \ + "$alarm_type" \ + "$target_instance_id" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "$(json_escape "${detail:0:ALARM_DETAIL_MAX_LENGTH}")")" + + # 토큰이 프로세스 목록에 남지 않도록 헤더를 설정 파일로 전달합니다. + header_config="$(mktemp)" + chmod 600 "$header_config" + printf 'header = "X-Internal-Alarm-Token: %s"\n' "$ALARM_API_TOKEN" >"$header_config" + + for port in ${ALARM_API_PORTS}; do + if curl -fsS \ + --config "$header_config" \ + --max-time "$ALARM_TIMEOUT_SECONDS" \ + --retry "$ALARM_RETRY_COUNT" \ + --retry-delay 3 \ + -X POST "http://${ALARM_API_HOST}:${port}${ALARM_PATH}" \ + -H "Content-Type: application/json" \ + -d "$payload" >/dev/null 2>&1; then + rm -f "$header_config" + alarm_sent=true + echo "Sent a backup alarm: type=$alarm_type port=$port" + return 0 + fi + done + + rm -f "$header_config" + echo "Failed to send a backup alarm: type=$alarm_type" >&2 + return 0 +} + +fail_with_alarm() { + local alarm_type="$1" + local detail="$2" + + echo "$detail" >&2 + send_backup_alarm "$alarm_type" "$detail" + exit 1 +} + +# 명시적으로 처리하지 않은 실패도 알리기 위해 스크립트 종료 시점에 한 번 더 확인합니다. +alarm_on_unexpected_failure() { + local exit_code=$? + local default_alarm_type="$1" + + if ((exit_code != 0)) && [[ "$alarm_sent" != "true" ]]; then + send_backup_alarm "$default_alarm_type" "unexpected failure with exit code $exit_code" + fi + return 0 +} + +# 스크립트는 돌고 있지만 업로드가 계속 실패해 마지막 성공이 오래된 경우를 알립니다. +# EC2 나 타이머 자체가 멈춘 경우는 이 방식으로 감지할 수 없어 외부 모니터링이 필요합니다. +alarm_if_upload_delayed() { + local success_file="$1" + local threshold_seconds="$2" + local last_success_epoch + local elapsed_seconds + + [[ -s "$success_file" ]] || return 0 + last_success_epoch="$(<"$success_file")" + [[ "$last_success_epoch" =~ ^[0-9]+$ ]] || return 0 + + elapsed_seconds=$(( $(date -u +%s) - last_success_epoch )) + if ((elapsed_seconds > threshold_seconds)); then + send_backup_alarm BINLOG_UPLOAD_DELAYED \ + "the last successful binlog upload was $elapsed_seconds seconds ago" + # 지연은 실패가 아니므로, 이번 실행이 실제로 실패하면 다시 알릴 수 있도록 되돌립니다. + alarm_sent=false + fi +} diff --git a/scripts/mysql_backup/tests/run.sh b/scripts/mysql_backup/tests/run.sh index 4aa0934..a162f62 100755 --- a/scripts/mysql_backup/tests/run.sh +++ b/scripts/mysql_backup/tests/run.sh @@ -78,10 +78,13 @@ set -Eeuo pipefail readonly BACKUP_ROOT="$TEST_BACKUP_ROOT" readonly MYSQL_CONTAINER="mysql-server" require_backup_environment() { :; } +require_alarm_environment() { :; } require_commands() { :; } mountpoint() { :; } docker() { :; } aws() { :; } +# 알림 경로의 tcp 확인을 통과시킨다 +timeout() { return 0; } require_dump_staging_space() { printf '%s\n' '1024 9999999999 268437504'; } mysql_query() { if [[ "$1" == *'@@log_bin'* ]]; then @@ -102,6 +105,9 @@ EOF MYSQL_BACKUP_BUCKET="test-bucket" \ MYSQL_DATABASE="test_database" \ AWS_REGION="ap-northeast-2" \ + ALARM_API_HOST="172.31.0.10" \ + ALARM_API_PORTS="8080 9080" \ + ALARM_API_TOKEN="test-token" \ MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-validate" >/dev/null @@ -110,6 +116,9 @@ EOF MYSQL_BACKUP_BUCKET="test-bucket" \ MYSQL_DATABASE="missing_database" \ AWS_REGION="ap-northeast-2" \ + ALARM_API_HOST="172.31.0.10" \ + ALARM_API_PORTS="8080 9080" \ + ALARM_API_TOKEN="test-token" \ MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-validate" >/dev/null 2>&1; then echo "Validation must reject a missing backup database." >&2 @@ -136,6 +145,29 @@ readonly AWS_REGION="${AWS_REGION:-ap-northeast-2}" require_backup_environment() { :; } require_commands() { :; } flock() { return 0; } +curl() { return 0; } +instance_id() { printf '%s' 'i-test'; } +alarm_sent=false +send_backup_alarm() { + if [[ -n "${TEST_ALARM_LOG:-}" ]]; then + printf '%s\n' "$1" >>"$TEST_ALARM_LOG" + fi + alarm_sent=true + return 0 +} +fail_with_alarm() { + echo "$2" >&2 + send_backup_alarm "$1" "$2" + exit 1 +} +alarm_on_unexpected_failure() { + local exit_code=$? + if ((exit_code != 0)) && [[ "$alarm_sent" != "true" ]]; then + send_backup_alarm "$1" "unexpected failure with exit code $exit_code" + fi + return 0 +} +alarm_if_upload_delayed() { :; } aws() { printf '%s' "${TEST_S3_KEYS:-}"; } require_dump_staging_space() { if [[ -n "${TEST_SPACE_CHECK_LOG:-}" ]]; then @@ -500,6 +532,174 @@ EOF fi } +test_backup_alarm_port_fallback() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + export ALARM_API_HOST="172.31.0.10" + export ALARM_API_PORTS="8080 9080" + export ALARM_API_TOKEN="test-token" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + local attempt_log="$TEST_ROOT/alarm-attempts" + : >"$attempt_log" + instance_id() { printf '%s' 'i-test'; } + # 활성 슬롯만 응답하는 상황을 재현한다. blue 는 닫혀 있고 green 만 열려 있다. + curl() { + local argument + for argument in "$@"; do + case "$argument" in + http://*:8080/*) echo "8080" >>"$attempt_log"; return 7 ;; + http://*:9080/*) echo "9080" >>"$attempt_log"; return 0 ;; + esac + done + return 0 + } + + send_backup_alarm DUMP_FAILED "test detail" >/dev/null + assert_equals \ + "8080 9080" \ + "$(tr '\n' ' ' <"$attempt_log" | sed 's/ $//')" \ + "the alarm must try the blue port first and fall back to the green port" + assert_equals "true" "$alarm_sent" "a delivered alarm must mark the sent flag" + ) +} + +test_backup_alarm_failure_does_not_break_backup() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + export ALARM_API_HOST="172.31.0.10" + export ALARM_API_PORTS="8080 9080" + export ALARM_API_TOKEN="test-token" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + instance_id() { printf '%s' 'i-test'; } + curl() { return 7; } + + if ! send_backup_alarm DUMP_FAILED "test detail" >/dev/null 2>&1; then + echo "An alarm delivery failure must not fail the backup." >&2 + exit 1 + fi + assert_equals "false" "$alarm_sent" "an undelivered alarm must not mark the sent flag" + ) +} + +test_backup_alarm_skipped_without_configuration() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + curl() { echo "The alarm must not be sent without configuration." >&2; return 99; } + instance_id() { printf '%s' 'i-test'; } + + send_backup_alarm DUMP_FAILED "test detail" >/dev/null 2>&1 + assert_equals "false" "$alarm_sent" "an alarm without configuration must not be marked as sent" + ) +} + +test_backup_alarm_detail_escaping() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + assert_equals \ + 'say \"hi\"' \ + "$(json_escape 'say "hi"')" \ + "double quotes in the detail must be escaped for json" + assert_equals \ + 'a\\b' \ + "$(json_escape 'a\b')" \ + "backslashes in the detail must be escaped for json" + assert_equals \ + 'first\nsecond' \ + "$(json_escape "$(printf 'first\nsecond')")" \ + "newlines in the detail must be escaped for json" + ) +} + +test_unexpected_failure_alarm_is_sent_once() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + export ALARM_API_HOST="172.31.0.10" + export ALARM_API_PORTS="8080" + export ALARM_API_TOKEN="test-token" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + local send_log="$TEST_ROOT/alarm-send-count" + : >"$send_log" + instance_id() { printf '%s' 'i-test'; } + curl() { echo "sent" >>"$send_log"; return 0; } + + # 이미 알린 실패에는 종료 시점 알림을 중복해서 보내지 않는다. + send_backup_alarm DUMP_FAILED "explicit failure" >/dev/null + ( exit 1 ) || alarm_on_unexpected_failure DUMP_FAILED + assert_equals \ + "1" \ + "$(wc -l <"$send_log" | tr -d ' ')" \ + "an already reported failure must not be alarmed twice" + ) +} + +test_binlog_delay_alarm_threshold() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + local alarm_log="$TEST_ROOT/delay-alarm.log" + local success_file="$TEST_ROOT/last-binlog-success" + send_backup_alarm() { printf '%s\n' "$1" >>"$alarm_log"; alarm_sent=true; return 0; } + date() { + if [[ "$*" == "-u +%s" ]]; then + printf '%s\n' '1784170800' + else + command date "$@" + fi + } + + # 한 주기(5분)만 지난 상태는 정상 범위로 보고 알리지 않는다. + : >"$alarm_log" + printf '%s\n' '1784170500' >"$success_file" + alarm_if_upload_delayed "$success_file" 900 + assert_equals "" "$(cat "$alarm_log")" "a single missed cycle must not raise a delay alarm" + + # 세 주기를 넘기면 지연으로 알린다. + : >"$alarm_log" + printf '%s\n' '1784169600' >"$success_file" + alarm_if_upload_delayed "$success_file" 900 + assert_equals \ + "BINLOG_UPLOAD_DELAYED" \ + "$(cat "$alarm_log")" \ + "an upload delayed beyond three cycles must be alarmed" + assert_equals \ + "false" \ + "$alarm_sent" \ + "a delay alarm must not suppress the alarm for an actual failure in the same run" + + # 마지막 성공 기록이 없으면 판단하지 않는다. + : >"$alarm_log" + rm -f "$success_file" + alarm_if_upload_delayed "$success_file" 900 + assert_equals "" "$(cat "$alarm_log")" "a missing success record must not raise a delay alarm" + ) +} + test_upload_idempotency test_dump_space_calculation test_validate_requires_schema @@ -507,4 +707,10 @@ test_binlog_chain test_dump_retry_manifest test_dump_rejects_insufficient_space test_dump_discards_stale_job +test_backup_alarm_port_fallback +test_backup_alarm_failure_does_not_break_backup +test_backup_alarm_skipped_without_configuration +test_backup_alarm_detail_escaping +test_unexpected_failure_alarm_is_sent_once +test_binlog_delay_alarm_threshold echo "All MySQL backup tests passed." diff --git a/scripts/mysql_backup/validate-remote.sh b/scripts/mysql_backup/validate-remote.sh index 7d2f850..29f94fb 100755 --- a/scripts/mysql_backup/validate-remote.sh +++ b/scripts/mysql_backup/validate-remote.sh @@ -10,11 +10,11 @@ if ((EUID != 0)); then fi if [[ -x "$VALIDATE_BIN" && -f "$CONFIG_FILE" ]]; then - unset MYSQL_BACKUP_BUCKET MYSQL_DATABASE AWS_REGION + unset MYSQL_BACKUP_BUCKET MYSQL_DATABASE AWS_REGION ALARM_API_HOST ALARM_API_PORTS ALARM_API_TOKEN while IFS='=' read -r key value || [[ -n "$key" ]]; do [[ -z "$key" || "$key" == \#* ]] && continue case "$key" in - MYSQL_BACKUP_BUCKET|MYSQL_DATABASE|AWS_REGION) + MYSQL_BACKUP_BUCKET|MYSQL_DATABASE|AWS_REGION|ALARM_API_HOST|ALARM_API_PORTS|ALARM_API_TOKEN) printf -v "$key" '%s' "$value" export "$key" ;; @@ -31,7 +31,7 @@ if [[ -x "$VALIDATE_BIN" && -f "$CONFIG_FILE" ]]; then exit 0 fi -for command_name in aws docker flock gzip sha256sum; do +for command_name in aws curl docker flock gzip sha256sum timeout; do command -v "$command_name" >/dev/null || { echo "Required command is not installed: $command_name" >&2 exit 1 @@ -40,6 +40,17 @@ done : "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET is required for pre-installation validation}" : "${MYSQL_DATABASE:?MYSQL_DATABASE is required for pre-installation validation}" : "${AWS_REGION:?AWS_REGION is required for pre-installation validation}" +: "${ALARM_API_HOST:?ALARM_API_HOST is required for pre-installation validation}" +: "${ALARM_API_PORTS:?ALARM_API_PORTS is required for pre-installation validation}" +: "${ALARM_API_TOKEN:?ALARM_API_TOKEN is required for pre-installation validation}" +if [[ ! "$ALARM_API_HOST" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "Invalid alarm api host." >&2 + exit 1 +fi +if [[ ! "$ALARM_API_PORTS" =~ ^[0-9]+( [0-9]+)*$ ]]; then + echo "Invalid alarm api ports." >&2 + exit 1 +fi if [[ ! "$MYSQL_BACKUP_BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]]; then echo "Invalid S3 bucket name." >&2 exit 1 @@ -86,5 +97,20 @@ if ((available_bytes < required_bytes)); then exit 1 fi aws s3api head-bucket --bucket "$MYSQL_BACKUP_BUCKET" --region "$AWS_REGION" >/dev/null + +# 비활성 슬롯은 내려가 있으므로 설정된 포트 중 하나라도 열려 있으면 통과합니다. +alarm_endpoint_reachable=false +for alarm_port in ${ALARM_API_PORTS}; do + if timeout 3 bash -c "exec 3<>/dev/tcp/${ALARM_API_HOST}/${alarm_port}" 2>/dev/null; then + alarm_endpoint_reachable=true + echo "Alarm api is reachable on port $alarm_port." + break + fi +done +if [[ "$alarm_endpoint_reachable" != "true" ]]; then + echo "Alarm api is not reachable on any of the configured ports: $ALARM_API_PORTS" >&2 + exit 1 +fi + df -h / /mnt/mysql-data echo "Pre-installation validation succeeded." From 90082eaa0ee9a766ce0365d672d7cbb25a9fb212 Mon Sep 17 00:00:00 2001 From: Hexeong <123macanic@naver.com> Date: Thu, 20 Aug 2026 21:00:35 +0900 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=EB=B0=B1=EC=97=85=20=EB=B2=84?= =?UTF-8?q?=ED=82=B7=EC=9D=98=20SSE-C=20=EC=B0=A8=EB=8B=A8=EC=9D=84=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=EC=97=90=20=EA=B3=A0=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 선언이 없어 apply 시 SSE-C 차단이 해제되는 drift 를 제거한다 Co-Authored-By: Claude Opus 5 --- environment/prod/mysql_backup.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environment/prod/mysql_backup.tf b/environment/prod/mysql_backup.tf index 2bb5c55..508a336 100644 --- a/environment/prod/mysql_backup.tf +++ b/environment/prod/mysql_backup.tf @@ -58,6 +58,9 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "mysql_backup" { bucket = aws_s3_bucket.mysql_backup.id rule { + # 백업은 SSE-S3로 고정합니다. 선언하지 않으면 apply 시 SSE-C 차단이 해제됩니다. + blocked_encryption_types = ["SSE-C"] + apply_server_side_encryption_by_default { sse_algorithm = "AES256" } From 558f5c7703a2556df3829ccc66f09794e3a2bb16 Mon Sep 17 00:00:00 2001 From: Hexeong <123macanic@naver.com> Date: Thu, 20 Aug 2026 21:36:34 +0900 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=EB=B0=B1=EC=97=85=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=20=EB=A6=AC=EB=B7=B0=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 사전 조건 검사 실패도 알리도록 종료 트랩을 소싱 직후로 옮긴다 - validate 모드의 인증 토큰을 표준 입력으로만 전달해 원격 프로세스 인자와 sudo 감사 로그에 남지 않게 한다 - 알림 대상 주소와 포트를 유효 범위까지 검증한다 - 알림 준비 단계가 실패해도 백업이 실패하지 않도록 각 단계를 처리한다 - blocked_encryption_types 를 지원하는 aws provider 버전으로 제약을 올린다 - 지연 임계값과 토큰 회전 절차를 구현과 일치하도록 문서를 고친다 Co-Authored-By: Claude Opus 5 --- .github/workflows/mysql-backup-deploy.yml | 14 ++++-- environment/prod/provider.tf | 5 ++- scripts/mysql_backup/README.md | 24 +++++++++-- scripts/mysql_backup/bin/mysql-backup-binlog | 6 +-- scripts/mysql_backup/bin/mysql-backup-dump | 6 +-- scripts/mysql_backup/lib/backup-common.sh | 45 +++++++++++++++++--- scripts/mysql_backup/tests/run.sh | 40 +++++++++++++++++ scripts/mysql_backup/validate-remote.sh | 18 +++++++- 8 files changed, 134 insertions(+), 24 deletions(-) diff --git a/.github/workflows/mysql-backup-deploy.yml b/.github/workflows/mysql-backup-deploy.yml index 8567682..3116893 100644 --- a/.github/workflows/mysql-backup-deploy.yml +++ b/.github/workflows/mysql-backup-deploy.yml @@ -229,10 +229,16 @@ jobs: --instance-id "$DB_INSTANCE_ID" \ --instance-os-user ubuntu \ --ssh-public-key "file://$KEY_DIR/id_ed25519.pub" >/dev/null - REMOTE_VALIDATE_COMMAND="env MYSQL_BACKUP_BUCKET=$(printf '%q' "$MYSQL_BACKUP_BUCKET") MYSQL_DATABASE=$(printf '%q' "$MYSQL_DATABASE") AWS_REGION=$(printf '%q' "$AWS_REGION") ALARM_API_HOST=$(printf '%q' "$ALARM_API_HOST") ALARM_API_PORTS=$(printf '%q' "$ALARM_API_PORTS") ALARM_API_TOKEN=$(printf '%q' "$ALARM_API_TOKEN") bash -s" - ssh "${SSH_OPTIONS[@]}" ubuntu@127.0.0.1 \ - "sudo bash -c $(printf '%q' "$REMOTE_VALIDATE_COMMAND")" \ - < scripts/mysql_backup/validate-remote.sh + # 토큰이 원격 프로세스 인자와 sudo 감사 로그에 남지 않도록 표준 입력으로만 전달한다. + { + printf 'export MYSQL_BACKUP_BUCKET=%q\n' "$MYSQL_BACKUP_BUCKET" + printf 'export MYSQL_DATABASE=%q\n' "$MYSQL_DATABASE" + printf 'export AWS_REGION=%q\n' "$AWS_REGION" + printf 'export ALARM_API_HOST=%q\n' "$ALARM_API_HOST" + printf 'export ALARM_API_PORTS=%q\n' "$ALARM_API_PORTS" + printf 'export ALARM_API_TOKEN=%q\n' "$ALARM_API_TOKEN" + cat scripts/mysql_backup/validate-remote.sh + } | ssh "${SSH_OPTIONS[@]}" ubuntu@127.0.0.1 "sudo bash -s" exit 0 fi diff --git a/environment/prod/provider.tf b/environment/prod/provider.tf index 52f4aec..998c321 100644 --- a/environment/prod/provider.tf +++ b/environment/prod/provider.tf @@ -3,8 +3,9 @@ terraform { required_providers { aws = { - source = "hashicorp/aws" - version = ">= 5.0" + source = "hashicorp/aws" + # blocked_encryption_types 는 6.22.0 부터 지원합니다. + version = ">= 6.22.0" } mysql = { source = "petoju/mysql" diff --git a/scripts/mysql_backup/README.md b/scripts/mysql_backup/README.md index b08a7b7..4b81d73 100644 --- a/scripts/mysql_backup/README.md +++ b/scripts/mysql_backup/README.md @@ -48,15 +48,33 @@ DB EC2 (private subnet, 인터넷 경로 없음) | `DUMP_FAILED` | 여유 공간 부족, mysqldump 실패, 복구 기준점 누락, dump 업로드 실패 | | `BINLOG_UPLOAD_FAILED` | binlog 회전 실패, binlog 업로드 실패 | | `BINLOG_GAP_DETECTED` | binlog 번호 불연속, 번호 역행, 닫힌 파일 누락 | -| `BINLOG_UPLOAD_DELAYED` | 마지막 성공 업로드가 5분을 초과 | +| `BINLOG_UPLOAD_DELAYED` | 마지막 성공 업로드가 900초(타이머 3주기)를 초과 | + +`BINLOG_UPLOAD_DELAYED`의 임계값을 타이머 주기와 같은 300초로 두면 정상 동작 중에도 경계에서 매번 지연으로 판정되므로 3주기인 900초를 사용합니다. 판정은 binlog 작업이 실행되는 시점에 이루어지므로 실제 알림은 다음 실행에서 발생할 수 있습니다. `BINLOG_UPLOAD_DELAYED`는 스크립트가 실행되고 있을 때만 감지할 수 있습니다. EC2나 타이머 자체가 멈춘 경우는 감지할 수 없으므로 S3의 마지막 객체 시각을 외부에서 관찰하는 모니터링이 별도로 필요합니다. ### 알림 인증 토큰 -호출자 인증 토큰은 `config/secrets/prod_db.tfvars`의 `mysql_backup_fail_alarm_request_token`을 단일 원천으로 사용합니다. 배포 워크플로우가 secrets submodule에서 값을 읽어 DB EC2의 `/etc/solid-connection/mysql-backup.env`에 기록하므로, 토큰을 변경할 때 Terraform apply 없이 배포 워크플로우만 다시 실행하면 됩니다. +호출자 인증 토큰은 두 곳에서 읽습니다. 각 구성 요소가 자기 설정 체계를 따르므로 값 자체는 두 곳에 존재합니다. + +| 사용처 | 위치 | +|--------|------| +| DB EC2의 백업 스크립트 | `config/secrets/prod_db.tfvars`의 `mysql_backup_fail_alarm_request_token` | +| API 서버 | Parameter Store의 `/solid-connection/{env}/internal-alarm.token` | + +배포 워크플로우가 secrets submodule에서 값을 읽어 DB EC2의 `/etc/solid-connection/mysql-backup.env`에 기록하므로, 스크립트 쪽 값을 바꿀 때 Terraform apply는 필요하지 않습니다. + +### 토큰 회전 절차 + +두 곳의 값이 어긋나면 모든 알림이 401로 거부되므로 다음 순서를 지킵니다. + +1. Parameter Store의 `/solid-connection/{env}/internal-alarm.token`을 새 값으로 변경합니다. +2. API 서버를 재배포해 새 토큰을 읽게 합니다. +3. `config/secrets/prod_db.tfvars`의 `mysql_backup_fail_alarm_request_token`을 같은 값으로 변경하고 커밋합니다. +4. `MySQL Backup Deploy` 워크플로우를 `install`로 실행해 DB EC2의 환경 파일을 갱신합니다. -> API 서버는 같은 토큰을 Parameter Store의 `/solid-connection/{env}/internal-alarm.token`에서 읽습니다. 토큰을 회전할 때 두 곳을 함께 변경해야 하며, 한쪽만 변경하면 모든 알림이 401로 거부됩니다. +1번과 2번 사이에는 API 서버가 이전 토큰을 사용하므로 알림이 거부됩니다. 백업 자체는 계속 동작하며, 회전은 백업 실패가 없는 시점에 수행합니다. ## dump 실패 처리 diff --git a/scripts/mysql_backup/bin/mysql-backup-binlog b/scripts/mysql_backup/bin/mysql-backup-binlog index 5965a34..25bef51 100644 --- a/scripts/mysql_backup/bin/mysql-backup-binlog +++ b/scripts/mysql_backup/bin/mysql-backup-binlog @@ -5,15 +5,15 @@ readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql- # shellcheck source=../lib/backup-common.sh source "$LIB_DIR/backup-common.sh" +# 사전 조건 검사 실패도 알리도록 소싱 직후에 등록합니다. +trap 'alarm_on_unexpected_failure BINLOG_UPLOAD_FAILED' EXIT + require_backup_environment require_commands aws curl docker flock sha256sum # 타이머 주기(5분)와 같은 값을 쓰면 정상 동작 중에도 경계에서 매번 지연으로 판정되므로 3주기로 둔다. readonly UPLOAD_DELAY_THRESHOLD_SECONDS=900 -# 명시적으로 처리하지 않은 실패도 알리도록 등록합니다. -trap 'alarm_on_unexpected_failure BINLOG_UPLOAD_FAILED' EXIT - alarm_if_upload_delayed "$STATE_DIR/last-binlog-success" "$UPLOAD_DELAY_THRESHOLD_SECONDS" exec 9>"$STATE_DIR/binlog.lock" diff --git a/scripts/mysql_backup/bin/mysql-backup-dump b/scripts/mysql_backup/bin/mysql-backup-dump index f0f6931..1c60cdb 100755 --- a/scripts/mysql_backup/bin/mysql-backup-dump +++ b/scripts/mysql_backup/bin/mysql-backup-dump @@ -5,14 +5,14 @@ readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql- # shellcheck source=../lib/backup-common.sh source "$LIB_DIR/backup-common.sh" +# 사전 조건 검사 실패도 알리도록 소싱 직후에 등록합니다. +trap 'alarm_on_unexpected_failure DUMP_FAILED' EXIT + require_backup_environment require_commands aws curl docker flock gzip sha256sum readonly MAX_DUMP_JOB_AGE_SECONDS=21600 -# 명시적으로 처리하지 않은 실패도 알리도록 등록합니다. -trap 'alarm_on_unexpected_failure DUMP_FAILED' EXIT - exec 9>"$STATE_DIR/dump.lock" if ! flock -n 9; then echo "Another mysqldump backup is already running." >&2 diff --git a/scripts/mysql_backup/lib/backup-common.sh b/scripts/mysql_backup/lib/backup-common.sh index c3cb0b1..c6bc739 100644 --- a/scripts/mysql_backup/lib/backup-common.sh +++ b/scripts/mysql_backup/lib/backup-common.sh @@ -143,14 +143,37 @@ require_alarm_environment() { : "${ALARM_API_PORTS:?ALARM_API_PORTS is required}" : "${ALARM_API_TOKEN:?ALARM_API_TOKEN is required}" - if [[ ! "$ALARM_API_HOST" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then - echo "Invalid alarm api host." >&2 + validate_alarm_target "$ALARM_API_HOST" "$ALARM_API_PORTS" +} + +# 8진수로 해석되지 않도록 10# 을 붙여 비교합니다. +validate_alarm_target() { + local host="$1" + local ports="$2" + local octet + local port + + if [[ ! "$host" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "Invalid alarm api host: $host" >&2 return 1 fi - if [[ ! "$ALARM_API_PORTS" =~ ^[0-9]+( [0-9]+)*$ ]]; then - echo "Invalid alarm api ports." >&2 + for octet in ${host//./ }; do + if ((10#$octet > 255)); then + echo "Invalid alarm api host: $host" >&2 + return 1 + fi + done + + if [[ ! "$ports" =~ ^[0-9]+( [0-9]+)*$ ]]; then + echo "Invalid alarm api ports: $ports" >&2 return 1 fi + for port in $ports; do + if ((10#$port < 1 || 10#$port > 65535)); then + echo "Invalid alarm api ports: $ports" >&2 + return 1 + fi + done } instance_id() { @@ -198,9 +221,17 @@ send_backup_alarm() { "$(json_escape "${detail:0:ALARM_DETAIL_MAX_LENGTH}")")" # 토큰이 프로세스 목록에 남지 않도록 헤더를 설정 파일로 전달합니다. - header_config="$(mktemp)" - chmod 600 "$header_config" - printf 'header = "X-Internal-Alarm-Token: %s"\n' "$ALARM_API_TOKEN" >"$header_config" + # 준비 단계가 실패해도 백업 자체는 계속되어야 하므로 항상 0 으로 돌아갑니다. + if ! header_config="$(mktemp 2>/dev/null)"; then + echo "Failed to create a temporary file for the backup alarm request." >&2 + return 0 + fi + if ! chmod 600 "$header_config" 2>/dev/null \ + || ! printf 'header = "X-Internal-Alarm-Token: %s"\n' "$ALARM_API_TOKEN" >"$header_config" 2>/dev/null; then + echo "Failed to prepare the backup alarm request." >&2 + rm -f "$header_config" + return 0 + fi for port in ${ALARM_API_PORTS}; do if curl -fsS \ diff --git a/scripts/mysql_backup/tests/run.sh b/scripts/mysql_backup/tests/run.sh index a162f62..18ce823 100755 --- a/scripts/mysql_backup/tests/run.sh +++ b/scripts/mysql_backup/tests/run.sh @@ -700,6 +700,45 @@ test_binlog_delay_alarm_threshold() { ) } +test_alarm_target_validation() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + if ! validate_alarm_target "172.31.56.245" "8080 9080" 2>/dev/null; then + echo "A valid alarm target must pass validation." >&2 + exit 1 + fi + + # 옥텟 범위를 넘는 주소는 형식만 맞아도 거부한다. + if validate_alarm_target "999.999.999.999" "8080" 2>/dev/null; then + echo "An out-of-range octet must be rejected." >&2 + exit 1 + fi + if validate_alarm_target "172.31.56" "8080" 2>/dev/null; then + echo "An incomplete address must be rejected." >&2 + exit 1 + fi + + # 포트 범위를 벗어나면 거부한다. + if validate_alarm_target "172.31.56.245" "0" 2>/dev/null; then + echo "Port 0 must be rejected." >&2 + exit 1 + fi + if validate_alarm_target "172.31.56.245" "65536" 2>/dev/null; then + echo "Port 65536 must be rejected." >&2 + exit 1 + fi + if validate_alarm_target "172.31.56.245" "8080 70000" 2>/dev/null; then + echo "An out-of-range port in the list must be rejected." >&2 + exit 1 + fi + ) +} + test_upload_idempotency test_dump_space_calculation test_validate_requires_schema @@ -713,4 +752,5 @@ test_backup_alarm_skipped_without_configuration test_backup_alarm_detail_escaping test_unexpected_failure_alarm_is_sent_once test_binlog_delay_alarm_threshold +test_alarm_target_validation echo "All MySQL backup tests passed." diff --git a/scripts/mysql_backup/validate-remote.sh b/scripts/mysql_backup/validate-remote.sh index 29f94fb..4ae330f 100755 --- a/scripts/mysql_backup/validate-remote.sh +++ b/scripts/mysql_backup/validate-remote.sh @@ -43,14 +43,28 @@ done : "${ALARM_API_HOST:?ALARM_API_HOST is required for pre-installation validation}" : "${ALARM_API_PORTS:?ALARM_API_PORTS is required for pre-installation validation}" : "${ALARM_API_TOKEN:?ALARM_API_TOKEN is required for pre-installation validation}" +# 설치 전에는 공용 라이브러리가 없으므로 같은 범위 검증을 여기에 둡니다. +# 8진수로 해석되지 않도록 10# 을 붙여 비교합니다. if [[ ! "$ALARM_API_HOST" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then - echo "Invalid alarm api host." >&2 + echo "Invalid alarm api host: $ALARM_API_HOST" >&2 exit 1 fi +for alarm_host_octet in ${ALARM_API_HOST//./ }; do + if ((10#$alarm_host_octet > 255)); then + echo "Invalid alarm api host: $ALARM_API_HOST" >&2 + exit 1 + fi +done if [[ ! "$ALARM_API_PORTS" =~ ^[0-9]+( [0-9]+)*$ ]]; then - echo "Invalid alarm api ports." >&2 + echo "Invalid alarm api ports: $ALARM_API_PORTS" >&2 exit 1 fi +for alarm_port in $ALARM_API_PORTS; do + if ((10#$alarm_port < 1 || 10#$alarm_port > 65535)); then + echo "Invalid alarm api ports: $ALARM_API_PORTS" >&2 + exit 1 + fi +done if [[ ! "$MYSQL_BACKUP_BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]]; then echo "Invalid S3 bucket name." >&2 exit 1