Skip to content

Commit 593c525

Browse files
fix(health): avoid permanent DEGRADED from historical deadlocks
1 parent 4c93556 commit 593c525

2 files changed

Lines changed: 13 additions & 46 deletions

File tree

pom.xml

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -291,12 +291,6 @@
291291
<nvdApiServerId>nvd</nvdApiServerId>
292292
</configuration>
293293
</plugin>
294-
<plugin>
295-
<groupId>org.apache.maven.plugins</groupId>
296-
<artifactId>maven-jar-plugin</artifactId>
297-
<version>3.0.2</version>
298-
</plugin>
299-
300294
<plugin>
301295
<groupId>org.apache.maven.plugins</groupId>
302296
<artifactId>maven-jar-plugin</artifactId>

src/main/java/com/iemr/admin/service/health/HealthService.java

Lines changed: 13 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,6 @@ public class HealthService {
8585
private volatile AdvancedCheckResult cachedAdvancedCheckResult = null;
8686
private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock();
8787

88-
private volatile boolean deadlockCheckDisabled = false;
89-
9088
private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true;
9189

9290
public HealthService(DataSource dataSource,
@@ -179,7 +177,7 @@ private HealthCheckResult checkMySQLHealthSync() {
179177

180178
try (ResultSet rs = stmt.executeQuery()) {
181179
if (rs.next()) {
182-
boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection);
180+
boolean isDegraded = performAdvancedMySQLChecksWithThrottle();
183181
return new HealthCheckResult(true, null, isDegraded);
184182
}
185183
}
@@ -304,7 +302,7 @@ private String computeOverallStatus(Map<String, Map<String, Object>> components)
304302
return STATUS_UP;
305303
}
306304

307-
private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) {
305+
private boolean performAdvancedMySQLChecksWithThrottle() {
308306
if (!ADVANCED_HEALTH_CHECKS_ENABLED) {
309307
return false;
310308
}
@@ -328,12 +326,17 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) {
328326
return cachedAdvancedCheckResult.isDegraded;
329327
}
330328

331-
AdvancedCheckResult result = performAdvancedMySQLChecks(connection);
332-
333-
lastAdvancedCheckTime = currentTime;
334-
cachedAdvancedCheckResult = result;
335-
336-
return result.isDegraded;
329+
// Acquire a fresh connection for advanced checks to avoid pool exhaustion
330+
try (Connection connection = dataSource.getConnection()) {
331+
AdvancedCheckResult result = performAdvancedMySQLChecks(connection);
332+
lastAdvancedCheckTime = currentTime;
333+
cachedAdvancedCheckResult = result;
334+
return result.isDegraded;
335+
}
336+
} catch (Exception e) {
337+
logger.debug("Failed to get connection for advanced checks: {}", e.getMessage());
338+
// Return cached result or false if no cache
339+
return cachedAdvancedCheckResult != null ? cachedAdvancedCheckResult.isDegraded : false;
337340
} finally {
338341
advancedCheckLock.writeLock().unlock();
339342
}
@@ -348,10 +351,6 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) {
348351
hasIssues = true;
349352
}
350353

351-
if (hasDeadlocks(connection)) {
352-
logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK);
353-
hasIssues = true;
354-
}
355354

356355
if (hasSlowQueries(connection)) {
357356
logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES);
@@ -390,32 +389,6 @@ private boolean hasLockWaits(Connection connection) {
390389
return false;
391390
}
392391

393-
private boolean hasDeadlocks(Connection connection) {
394-
if (deadlockCheckDisabled) {
395-
return false;
396-
}
397-
398-
try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) {
399-
stmt.setQueryTimeout(2);
400-
try (ResultSet rs = stmt.executeQuery()) {
401-
if (rs.next()) {
402-
String innodbStatus = rs.getString(3);
403-
return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK");
404-
}
405-
}
406-
} catch (java.sql.SQLException e) {
407-
if (e.getErrorCode() == 1142 || e.getErrorCode() == 1227) {
408-
deadlockCheckDisabled = true;
409-
logger.warn("Deadlock check disabled: Insufficient privileges");
410-
} else {
411-
logger.debug("Could not check for deadlocks");
412-
}
413-
} catch (Exception e) {
414-
logger.debug("Could not check for deadlocks");
415-
}
416-
return false;
417-
}
418-
419392
private boolean hasSlowQueries(Connection connection) {
420393
try (PreparedStatement stmt = connection.prepareStatement(
421394
"SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " +

0 commit comments

Comments
 (0)