-
Notifications
You must be signed in to change notification settings - Fork 8
feat: MySQL 백업 실패 알림 내부 전용 API 구현 #833
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
11 changes: 11 additions & 0 deletions
11
src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package com.example.solidconnection.alarm.config; | ||
|
|
||
| import org.springframework.boot.context.properties.ConfigurationProperties; | ||
|
|
||
| @ConfigurationProperties(prefix = "discord.db-backup-fail-alarm") | ||
| public record DbBackupAlarmProperties( | ||
| String webhookUrl, | ||
| String mentionRoleId | ||
| ) { | ||
|
|
||
| } | ||
10 changes: 10 additions & 0 deletions
10
src/main/java/com/example/solidconnection/alarm/config/InternalAlarmAuthProperties.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package com.example.solidconnection.alarm.config; | ||
|
|
||
| import org.springframework.boot.context.properties.ConfigurationProperties; | ||
|
|
||
| @ConfigurationProperties(prefix = "internal-alarm") | ||
| public record InternalAlarmAuthProperties( | ||
| String token | ||
| ) { | ||
|
|
||
| } |
32 changes: 32 additions & 0 deletions
32
src/main/java/com/example/solidconnection/alarm/controller/DbBackupAlarmController.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| package com.example.solidconnection.alarm.controller; | ||
|
|
||
| import com.example.solidconnection.alarm.dto.DbBackupAlarmRequest; | ||
| import com.example.solidconnection.alarm.service.DbBackupAlarmService; | ||
| import jakarta.validation.Valid; | ||
| import lombok.RequiredArgsConstructor; | ||
| import org.springframework.http.ResponseEntity; | ||
| import org.springframework.web.bind.annotation.PostMapping; | ||
| import org.springframework.web.bind.annotation.RequestBody; | ||
| import org.springframework.web.bind.annotation.RequestHeader; | ||
| import org.springframework.web.bind.annotation.RequestMapping; | ||
| import org.springframework.web.bind.annotation.RestController; | ||
|
|
||
| @RestController | ||
| @RequestMapping("/internal/alarms") | ||
| @RequiredArgsConstructor | ||
| public class DbBackupAlarmController { | ||
|
|
||
| private static final String INTERNAL_ALARM_TOKEN_HEADER = "X-Internal-Alarm-Token"; | ||
|
|
||
| private final DbBackupAlarmService dbBackupAlarmService; | ||
|
|
||
| // DB EC2 의 백업 실패 이벤트를 받아 Discord 로 알리는 내부 전용 api | ||
| @PostMapping("/db-backup") | ||
| public ResponseEntity<Void> alarmBackupFailure( | ||
| @RequestHeader(value = INTERNAL_ALARM_TOKEN_HEADER, required = false) String token, | ||
| @Valid @RequestBody DbBackupAlarmRequest dbBackupAlarmRequest | ||
| ) { | ||
| dbBackupAlarmService.alarmBackupFailure(token, dbBackupAlarmRequest); | ||
| return ResponseEntity.accepted().build(); | ||
| } | ||
| } |
19 changes: 19 additions & 0 deletions
19
src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| package com.example.solidconnection.alarm.domain; | ||
|
|
||
| import lombok.Getter; | ||
|
|
||
| @Getter | ||
| public enum DbBackupAlarmType { | ||
|
|
||
| DUMP_FAILED("전체 덤프 실패"), | ||
| BINLOG_UPLOAD_FAILED("바이너리 로그 업로드 실패"), | ||
| BINLOG_GAP_DETECTED("바이너리 로그 누락"), | ||
| BINLOG_UPLOAD_DELAYED("바이너리 로그 업로드 지연"), | ||
| ; | ||
|
|
||
| private final String displayName; | ||
|
|
||
| DbBackupAlarmType(String displayName) { | ||
| this.displayName = displayName; | ||
| } | ||
| } |
25 changes: 25 additions & 0 deletions
25
src/main/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| package com.example.solidconnection.alarm.dto; | ||
|
|
||
| import com.example.solidconnection.alarm.domain.DbBackupAlarmType; | ||
| import jakarta.validation.constraints.NotBlank; | ||
| import jakarta.validation.constraints.NotNull; | ||
| import jakarta.validation.constraints.Size; | ||
| import java.time.Instant; | ||
|
|
||
| public record DbBackupAlarmRequest( | ||
|
|
||
| @NotNull | ||
| DbBackupAlarmType type, | ||
|
|
||
| @NotBlank | ||
| @Size(max = 64) | ||
| String instanceId, | ||
|
|
||
| @NotNull | ||
| Instant occurredAt, | ||
|
|
||
| @Size(max = 1000) | ||
| String detail | ||
| ) { | ||
|
|
||
| } |
141 changes: 141 additions & 0 deletions
141
src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| package com.example.solidconnection.alarm.service; | ||
|
|
||
| import static com.example.solidconnection.common.exception.ErrorCode.INTERNAL_ALARM_UNAUTHORIZED; | ||
|
|
||
| import com.example.solidconnection.alarm.config.DbBackupAlarmProperties; | ||
| import com.example.solidconnection.alarm.config.InternalAlarmAuthProperties; | ||
| import com.example.solidconnection.alarm.dto.DbBackupAlarmRequest; | ||
| import com.example.solidconnection.common.discord.DiscordWebhookSender; | ||
| import com.example.solidconnection.common.exception.CustomException; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.security.MessageDigest; | ||
| import java.time.Duration; | ||
| import java.util.List; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.data.redis.core.RedisTemplate; | ||
| import org.springframework.stereotype.Service; | ||
|
|
||
| /* | ||
| * - DB EC2 는 private subnet 에 있어 Discord 로 직접 요청할 수 없다. | ||
| * - 따라서 백업 실패 이벤트를 전달받아 Discord 로 중계한다. | ||
| * */ | ||
| @Service | ||
| @RequiredArgsConstructor | ||
| @Slf4j | ||
| public class DbBackupAlarmService { | ||
|
|
||
| private static final String SUPPRESSION_KEY_PREFIX = "db-backup-alarm:"; | ||
| private static final Duration SUPPRESSION_TTL = Duration.ofMinutes(10); | ||
| private static final String ROLE_MENTION_FORMAT = "<@&%s>"; | ||
| private static final String EMPTY_DETAIL = "-"; | ||
|
|
||
| private final DiscordWebhookSender discordWebhookSender; | ||
| private final DbBackupAlarmProperties dbBackupAlarmProperties; | ||
| private final InternalAlarmAuthProperties internalAlarmAuthProperties; | ||
| private final RedisTemplate<String, String> redisTemplate; | ||
|
|
||
| @Value("${spring.profiles.active:}") | ||
| private String environment; | ||
|
|
||
| public void alarmBackupFailure(String token, DbBackupAlarmRequest request) { | ||
| validateToken(token); | ||
|
|
||
| String suppressionKey = buildSuppressionKey(request); | ||
| if (isSuppressed(suppressionKey)) { | ||
| return; | ||
| } | ||
| boolean isSent = discordWebhookSender.send( | ||
| dbBackupAlarmProperties.webhookUrl(), | ||
| buildMessage(request), | ||
| mentionableRoleIds() | ||
| ); | ||
| if (!isSent) { | ||
| releaseSuppression(suppressionKey); | ||
| } | ||
|
lsy1307 marked this conversation as resolved.
|
||
| } | ||
|
|
||
| /* | ||
| * - 토큰이 설정되지 않은 환경에서는 모든 요청을 거부한다. | ||
| * - 설정 누락과 토큰 불일치를 같은 응답으로 처리해 내부 상태가 드러나지 않게 한다. | ||
| * */ | ||
| private void validateToken(String token) { | ||
| String configuredToken = internalAlarmAuthProperties.token(); | ||
| if (configuredToken == null || configuredToken.isBlank()) { | ||
| log.error("내부 알림 인증 토큰이 설정되지 않아 요청을 거부했습니다."); | ||
| throw new CustomException(INTERNAL_ALARM_UNAUTHORIZED); | ||
| } | ||
| if (token == null || !MessageDigest.isEqual( | ||
| token.getBytes(StandardCharsets.UTF_8), | ||
| configuredToken.getBytes(StandardCharsets.UTF_8))) { | ||
| throw new CustomException(INTERNAL_ALARM_UNAUTHORIZED); | ||
| } | ||
| } | ||
|
|
||
| private String buildSuppressionKey(DbBackupAlarmRequest request) { | ||
| return SUPPRESSION_KEY_PREFIX + request.type().name() + ":" + request.instanceId(); | ||
| } | ||
|
|
||
| /* | ||
| * - 같은 유형과 인스턴스의 알림이 반복되면 일정 시간 동안 전송하지 않는다. | ||
| * - Redis 를 사용할 수 없을 때는 알림 누락을 막기 위해 억제하지 않는다. | ||
| * */ | ||
| private boolean isSuppressed(String suppressionKey) { | ||
| try { | ||
| Boolean isFirstAlarm = redisTemplate.opsForValue().setIfAbsent(suppressionKey, "1", SUPPRESSION_TTL); | ||
| return !Boolean.TRUE.equals(isFirstAlarm); | ||
| } catch (Exception e) { | ||
| log.error("백업 알림 중복 억제 상태를 확인하지 못해 알림을 그대로 전송합니다. key={}", suppressionKey, e); | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| private String buildMessage(DbBackupAlarmRequest request) { | ||
| return buildRoleMention() + "[%s] MySQL 백업 알림: %s\n인스턴스: %s\n발생 시각: %s\n상세: %s" | ||
| .formatted( | ||
| environment.toUpperCase(), | ||
| request.type().getDisplayName(), | ||
| request.instanceId(), | ||
| request.occurredAt(), | ||
| resolveDetail(request.detail()) | ||
| ); | ||
|
coderabbitai[bot] marked this conversation as resolved.
Outdated
|
||
| } | ||
|
|
||
| /* | ||
| * - 멘션할 역할이 설정되지 않으면 멘션 없이 알림만 보낸다. | ||
| * */ | ||
| private String buildRoleMention() { | ||
| String mentionRoleId = dbBackupAlarmProperties.mentionRoleId(); | ||
| if (mentionRoleId == null || mentionRoleId.isBlank()) { | ||
| return ""; | ||
| } | ||
| return ROLE_MENTION_FORMAT.formatted(mentionRoleId) + "\n"; | ||
| } | ||
|
|
||
| private String resolveDetail(String detail) { | ||
| if (detail == null || detail.isBlank()) { | ||
| return EMPTY_DETAIL; | ||
| } | ||
| return detail; | ||
| } | ||
|
|
||
| private List<String> mentionableRoleIds() { | ||
| String mentionRoleId = dbBackupAlarmProperties.mentionRoleId(); | ||
| if (mentionRoleId == null || mentionRoleId.isBlank()) { | ||
| return List.of(); | ||
| } | ||
| return List.of(mentionRoleId); | ||
| } | ||
|
|
||
| /* | ||
| * - 전송에 실패하면 억제 상태를 되돌려 다음 백업 주기의 알림이 막히지 않게 한다. | ||
| * */ | ||
| private void releaseSuppression(String suppressionKey) { | ||
| try { | ||
| redisTemplate.delete(suppressionKey); | ||
| } catch (Exception e) { | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
| log.error("백업 알림 중복 억제 상태를 해제하지 못했습니다. key={}", suppressionKey, e); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package com.example.solidconnection.common.discord; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Map; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.springframework.http.HttpEntity; | ||
| import org.springframework.http.HttpHeaders; | ||
| import org.springframework.http.MediaType; | ||
| import org.springframework.stereotype.Component; | ||
| import org.springframework.web.client.RestTemplate; | ||
|
|
||
| /* | ||
| * - Discord Webhook 으로 메시지를 전송한다. | ||
| * - 알림 전송 실패가 호출한 기능을 실패시키지 않도록 예외를 격리하고, 전송 여부를 반환해 후속 처리를 맡긴다. | ||
| * - 채널별로 webhook url 이 다르므로 url 을 인자로 받는다. | ||
| * */ | ||
| @Component | ||
| @RequiredArgsConstructor | ||
| @Slf4j | ||
| public class DiscordWebhookSender { | ||
|
|
||
| private final RestTemplate restTemplate; | ||
|
|
||
| public boolean send(String webhookUrl, String content) { | ||
| return send(webhookUrl, content, List.of()); | ||
| } | ||
|
|
||
| /* | ||
| * - mentionableRoleIds 에 지정한 역할만 멘션할 수 있다. | ||
| * - @everyone 과 @here 는 항상 차단되므로 content 에 섞여 들어와도 채널 전체를 호출하지 않는다. | ||
| * */ | ||
| public boolean send(String webhookUrl, String content, List<String> mentionableRoleIds) { | ||
| if (webhookUrl == null || webhookUrl.isBlank()) { | ||
| log.error("Discord webhook url 이 설정되지 않아 알림을 전송하지 못했습니다."); | ||
| return false; | ||
| } | ||
| try { | ||
| restTemplate.postForEntity(webhookUrl, buildRequest(content, mentionableRoleIds), Void.class); | ||
| return true; | ||
| } catch (Exception e) { | ||
| log.error("Discord 알림 전송에 실패했습니다.", e); | ||
| return false; | ||
| } | ||
| } | ||
|
lsy1307 marked this conversation as resolved.
|
||
|
|
||
| private HttpEntity<Map<String, Object>> buildRequest(String content, List<String> mentionableRoleIds) { | ||
| HttpHeaders headers = new HttpHeaders(); | ||
| headers.setContentType(MediaType.APPLICATION_JSON); | ||
| Map<String, Object> body = Map.of( | ||
| "content", content, | ||
| "allowed_mentions", Map.of( | ||
| "parse", List.of(), | ||
| "roles", mentionableRoleIds | ||
| ) | ||
| ); | ||
| return new HttpEntity<>(body, headers); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.