diff --git a/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh b/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh index ccac6353b30a..4ba09637f941 100755 --- a/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh +++ b/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh @@ -26,11 +26,48 @@ ifconfig netstat -i # Unit Tests -# - TEST_SET#1 runs install and test together so the module list must ensure no additional modules were tested -# due to the -am flag (include dependency modules) -# - tests for pinot-plugins should not be ran multi-threaded +# - Both test sets run plain `mvn test` (no install, no -am): the modules were already built +# and installed by .pinot_tests_build.sh, so only the modules listed here are tested. +# +# Parallelism / memory: +# - UNIT_TEST_FORK_COUNT (default 3) sets surefire forkCount so test *classes* run in +# separate parallel JVMs (reuseForks=false keeps one class per JVM). This is +# process-level isolation, not TestNG intra-JVM threading, so tests that were unsafe +# to run multi-threaded within a single JVM (e.g. pinot-plugins) are unaffected. +# Cross-fork resource collisions (ZK/controller ports, temp dirs) are avoided by +# offsetting per surefire.forkNumber; embedded Kafka clusters use ephemeral ports. +# This is the main lever for shortening the unit-test phase. 3 forks on the 4-vCPU +# runner keeps a core free for the Maven reactor / GC while test JVMs spend much of +# their time blocked on ZK/Helix/socket startup, so the extra fork still pays off. +# - UNIT_TEST_FORK_HEAP (default 2500m) caps per-fork heap so N forks fit in the +# runner's memory (N * heap + the mvn JVM must stay under the runner's RAM). +# - UNIT_TEST_RERUN_COUNT (default 0) retries a failing test before failing the build. Left at 0 +# because the load-sensitive flaky tests parallel forks exposed are fixed at the root cause +# (SegmentPreProcessorTest mtime granularity, LuceneMutableTextIndexTest NRT-refresh wait). It +# remains overridable as an escape hatch if a new flake appears, but is intentionally not a +# standing default so real failures are never masked. +UNIT_TEST_FORK_COUNT="${UNIT_TEST_FORK_COUNT:-3}" +# 2500m/fork: 3 forks * 2500m + the 2g Maven JVM (~9.5g) stays well under the runner's 16g. +UNIT_TEST_FORK_HEAP="${UNIT_TEST_FORK_HEAP:-2500m}" +UNIT_TEST_RERUN_COUNT="${UNIT_TEST_RERUN_COUNT:-0}" +# Coverage adds ~30% to the test phase (JaCoCo agent per fork + aggregate report). Keep it on by +# default to preserve Codecov behavior; set RUN_CODECOVERAGE=false (e.g. on PRs) to trade coverage +# for a faster run. +RUN_CODECOVERAGE="${RUN_CODECOVERAGE:-true}" +# Fork-scope the JaCoCo exec file (jacoco-.exec) so parallel forks don't append to +# one shared jacoco.exec and corrupt coverage. Only the unit lane sets this; other lanes keep +# the default empty suffix (target/jacoco.exec). +FORK_OPTS="-Dunit.test.fork.count=${UNIT_TEST_FORK_COUNT} -Dunit.test.fork.heap=${UNIT_TEST_FORK_HEAP} -Dunit.test.rerun.count=${UNIT_TEST_RERUN_COUNT} -Djacoco.exec.suffix=-\${surefire.forkNumber}" +if [ "$RUN_CODECOVERAGE" == "true" ]; then + COVERAGE_PROFILE=",codecoverage" +else + COVERAGE_PROFILE="" +fi if [ "$RUN_TEST_SET" == "1" ]; then - mvn test \ + # pinot-segment-local's tests run in set #2 to balance pinot-core's longer test time in this + # shard against set #2's longer build. It remains built in set #1 as a pinot-core dependency. + # No -am on this command, so only the listed modules test. + mvn test ${FORK_OPTS} \ -pl 'pinot-spi' \ -pl 'pinot-segment-spi' \ -pl 'pinot-common' \ @@ -38,10 +75,10 @@ if [ "$RUN_TEST_SET" == "1" ]; then -pl 'pinot-core' \ -pl 'pinot-query-planner' \ -pl 'pinot-query-runtime' \ - -P github-actions,codecoverage,no-integration-tests || exit 1 + -P github-actions,no-integration-tests${COVERAGE_PROFILE} || exit 1 fi if [ "$RUN_TEST_SET" == "2" ]; then - mvn test \ + mvn test ${FORK_OPTS} \ -pl '!pinot-spi' \ -pl '!pinot-segment-spi' \ -pl '!pinot-common' \ @@ -49,7 +86,13 @@ if [ "$RUN_TEST_SET" == "2" ]; then -pl '!pinot-query-planner' \ -pl '!pinot-query-runtime' \ -pl '!:pinot-yammer' \ - -P github-actions,codecoverage,no-integration-tests || exit 1 + -P github-actions,no-integration-tests${COVERAGE_PROFILE} || exit 1 fi -mvn jacoco:report-aggregate@report -P codecoverage || exit 1 +# Aggregate coverage across all per-fork exec files (jacoco-*.exec) written under forkCount>1, +# while still matching the single-fork jacoco.exec produced by non-parallel runs. Skipped when +# coverage is disabled. +if [ "$RUN_CODECOVERAGE" == "true" ]; then + mvn jacoco:report-aggregate@report -P codecoverage \ + -Djacoco.dataFileIncludes='**/target/jacoco-*.exec,**/target/jacoco.exec' || exit 1 +fi diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/queryquota/HelixExternalViewBasedQueryQuotaManagerTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/queryquota/HelixExternalViewBasedQueryQuotaManagerTest.java index e2efb05f45ec..5ba6072161f1 100644 --- a/pinot-broker/src/test/java/org/apache/pinot/broker/queryquota/HelixExternalViewBasedQueryQuotaManagerTest.java +++ b/pinot-broker/src/test/java/org/apache/pinot/broker/queryquota/HelixExternalViewBasedQueryQuotaManagerTest.java @@ -327,10 +327,10 @@ public void testWhenOnlyDefaultAppQuotaIsSetItAffectsAllApplications() Assert.assertEquals(_queryQuotaManager.getDatabaseRateLimiterMap().size(), 1); Assert.assertEquals(_queryQuotaManager.getApplicationRateLimiterMap().size(), 1); - runQueries(100, true, APP_NAME); - runQueries(100, true, "otherApp"); + assertApplicationRateLimitedInBurst(APP_NAME, 100); + assertApplicationRateLimitedInBurst("otherApp", 100); runQueries(100, false, "someApp"); - runQueries(201, true, "someApp"); + assertApplicationRateLimitedInBurst("someApp", 201); Assert.assertEquals(_queryQuotaManager.getApplicationRateLimiterMap().size(), 3); _queryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); @@ -697,4 +697,13 @@ private void runQueries(double qps, boolean shouldFail, String appName) Assert.assertTrue(failCount == 0, "Expected no failure with qps: " + qps + " and app :" + appName); } } + + private void assertApplicationRateLimitedInBurst(String appName, int numQueries) { + for (int i = 0; i < numQueries; i++) { + if (!_queryQuotaManager.acquireApplication(appName)) { + return; + } + } + Assert.fail("Expected application rate limiting for " + numQueries + " queries and app: " + appName); + } } diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index a6630836cbaf..a36aea1d9369 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -59,7 +59,8 @@ org.apache.maven.plugins maven-surefire-plugin - 1 + + ${unit.test.fork.count} true diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/ZkStarter.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/ZkStarter.java index 84bc8fcd6661..2672f8624112 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/utils/ZkStarter.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/ZkStarter.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -42,6 +43,33 @@ private ZkStarter() { public static final int DEFAULT_ZK_TEST_PORT = 2191; private static final int DEFAULT_ZK_CLIENT_RETRIES = 10; + /// Per-fork offset applied to the default test port so that concurrent surefire forks + /// (forkCount > 1, reuseForks=false) do not scan from the same base port and collide. + /// + /// This is purely a test-harness hook: the offset is derived from the `surefire.forkNumber` + /// system property, which surefire injects only inside a forked test JVM (1-based, so it is 1 + /// even at forkCount=1, and 1..N under parallel forks). In any production process that property + /// is absent, so `forkNumber()` returns 0 and the no-arg {@link #startLocalZkServer()} scans + /// from the historical {@link #DEFAULT_ZK_TEST_PORT}. Under tests each fork scans from a distinct + /// base ({@code DEFAULT_ZK_TEST_PORT + forkNumber*1000}); the exact port is still chosen by + /// findOpenPort and read back via {@code getZkUrl()}, so no caller depends on the literal base. + /// The stride (1000) is large enough that a fork exhausting ports below the next boundary + /// (findOpenPort scans upward) does not spill into the neighboring fork's band. + /// + /// The offset is deliberately centralized on the no-arg entry point rather than pushed into each + /// test: several tests across different modules call {@link #startLocalZkServer()} directly, and + /// duplicating the fork math into each caller (or introducing a parallel test-only start helper) + /// is more surface and more error-prone than one guarded, production-inert read here. + private static final int FORK_PORT_OFFSET = forkNumber() * 1000; + + private static int forkNumber() { + try { + return Integer.parseInt(System.getProperty("surefire.forkNumber", "0")); + } catch (NumberFormatException e) { + return 0; + } + } + public static class ZookeeperInstance { private PublicZooKeeperServerMain _serverMain; private String _dataDirPath; @@ -135,9 +163,10 @@ public void shutdown() { } } - /// Starts an empty local Zk instance on the default port + /// Starts an empty local Zk instance on the default port (offset per surefire fork so that + /// concurrent forks bind disjoint port ranges). public static ZookeeperInstance startLocalZkServer() { - return startLocalZkServer(NetUtils.findOpenPort(DEFAULT_ZK_TEST_PORT)); + return startLocalZkServer(NetUtils.findOpenPort(DEFAULT_ZK_TEST_PORT + FORK_PORT_OFFSET)); } public static String getDefaultZkStr() { @@ -147,8 +176,10 @@ public static String getDefaultZkStr() { /// Starts a local Zk instance with a generated empty data directory /// @param port The port to listen on public static ZookeeperInstance startLocalZkServer(final int port) { + // Use a random UUID rather than a timestamp so that concurrent forks/threads never share a + // ZK data directory (System.currentTimeMillis() collides when two instances start in the same ms). return startLocalZkServer(port, - org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis()); + org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + UUID.randomUUID()); } /// Starts a local Zk instance diff --git a/pinot-controller/pom.xml b/pinot-controller/pom.xml index 3fa8183c99b2..d89cf9834591 100644 --- a/pinot-controller/pom.xml +++ b/pinot-controller/pom.xml @@ -172,12 +172,41 @@ org.apache.maven.plugins maven-surefire-plugin + + ${project.build.directory}/surefire-reports/$${surefire.forkNumber} + + true + + + reporter + org.testng.reporters.JUnitReportReporter + + testng-statefull.xml testng-stateless.xml + + org.apache.maven.plugins + maven-surefire-report-plugin + ${surefire.version} + + + + ${project.build.directory}/surefire-reports/1/junitreports + ${project.build.directory}/surefire-reports/2/junitreports + ${project.build.directory}/surefire-reports/3/junitreports + + + diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterDynamicEnvTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterDynamicEnvTest.java index d81b380a7f70..7c876623485d 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterDynamicEnvTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterDynamicEnvTest.java @@ -43,6 +43,7 @@ /// This class tests env variables when starting controller from configs public class ControllerStarterDynamicEnvTest extends ControllerTest { private final Map _configOverride = new HashMap<>(); + private int _controllerPortOverride; @Override protected void overrideControllerConf(Map properties) { @@ -53,10 +54,11 @@ protected void overrideControllerConf(Map properties) { @Test public void testNoVariable() throws Exception { + int controllerPort = findControllerPortInForkRange(); _configOverride.clear(); _configOverride.put(CONTROLLER_HOST, "myHost"); _configOverride.put(CONFIG_OF_INSTANCE_ID, "Controller_myInstance"); - _configOverride.put(CONTROLLER_PORT, 1234); + _configOverride.put(CONTROLLER_PORT, controllerPort); startZk(); this.startController(); @@ -66,7 +68,7 @@ public void testNoVariable() InstanceConfig instanceConfig = HelixHelper.getInstanceConfig(_helixManager, instanceId); assertEquals(instanceConfig.getInstanceName(), instanceId); assertEquals(instanceConfig.getHostName(), "myHost"); - assertEquals(instanceConfig.getPort(), "1234"); + assertEquals(instanceConfig.getPort(), Integer.toString(controllerPort)); assertEquals(instanceConfig.getTags(), Set.of(CONTROLLER_INSTANCE)); stopController(); @@ -76,11 +78,12 @@ public void testNoVariable() @Test public void testOneVariable() throws Exception { + int controllerPort = findControllerPortInForkRange(); _configOverride.clear(); _configOverride.put("dynamic.env.config", "controller.host"); _configOverride.put(CONTROLLER_HOST, "HOST"); _configOverride.put(CONFIG_OF_INSTANCE_ID, "Controller_myInstance"); - _configOverride.put(CONTROLLER_PORT, 1234); + _configOverride.put(CONTROLLER_PORT, controllerPort); startZk(); this.startController(); @@ -90,7 +93,7 @@ public void testOneVariable() InstanceConfig instanceConfig = HelixHelper.getInstanceConfig(_helixManager, instanceId); assertEquals(instanceConfig.getInstanceName(), instanceId); assertEquals(instanceConfig.getHostName(), "myHost"); - assertEquals(instanceConfig.getPort(), "1234"); + assertEquals(instanceConfig.getPort(), Integer.toString(controllerPort)); assertEquals(instanceConfig.getTags(), Set.of(CONTROLLER_INSTANCE)); stopController(); @@ -100,6 +103,7 @@ public void testOneVariable() @Test public void testMultipleVariables() throws Exception { + int controllerPort = findControllerPortInForkRange(); _configOverride.clear(); _configOverride.put("dynamic.env.config", "controller.host,controller.port"); _configOverride.put(CONTROLLER_HOST, "HOST"); @@ -114,7 +118,7 @@ public void testMultipleVariables() InstanceConfig instanceConfig = HelixHelper.getInstanceConfig(_helixManager, instanceId); assertEquals(instanceConfig.getInstanceName(), instanceId); assertEquals(instanceConfig.getHostName(), "myHost"); - assertEquals(instanceConfig.getPort(), "1234"); + assertEquals(instanceConfig.getPort(), Integer.toString(controllerPort)); assertEquals(instanceConfig.getTags(), Set.of(CONTROLLER_INSTANCE)); stopController(); @@ -150,7 +154,7 @@ public void startController(Map properties) throws Exception { Map envVariables = new HashMap<>(); envVariables.put("HOST", "myHost"); - envVariables.put("PORT", "1234"); + envVariables.put("PORT", Integer.toString(_controllerPortOverride)); _controllerStarter = createControllerStarter(); _controllerStarter.init(new PinotConfiguration(properties, envVariables)); _controllerStarter.start(); @@ -184,4 +188,9 @@ public void startController(Map properties) } assertEquals(System.getProperty("user.timezone"), "UTC"); } + + private int findControllerPortInForkRange() { + _controllerPortOverride = NetUtils.findOpenPort(_nextControllerPort); + return _controllerPortOverride; + } } diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterStatelessTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterStatelessTest.java index 8ce575e03893..9b5c837aa74a 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterStatelessTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterStatelessTest.java @@ -24,6 +24,7 @@ import org.apache.helix.model.InstanceConfig; import org.apache.pinot.common.utils.helix.HelixHelper; import org.apache.pinot.controller.helix.ControllerTest; +import org.apache.pinot.spi.utils.NetUtils; import org.testng.annotations.Test; import static org.apache.pinot.controller.ControllerConf.CONTROLLER_HOST; @@ -46,10 +47,11 @@ protected void overrideControllerConf(Map properties) { @Test public void testHostnamePortOverride() throws Exception { + int controllerPort = NetUtils.findOpenPort(_nextControllerPort); _configOverride.clear(); _configOverride.put(CONFIG_OF_INSTANCE_ID, "Controller_myInstance"); _configOverride.put(CONTROLLER_HOST, "myHost"); - _configOverride.put(CONTROLLER_PORT, 1234); + _configOverride.put(CONTROLLER_PORT, controllerPort); startZk(); startController(); @@ -59,7 +61,7 @@ public void testHostnamePortOverride() InstanceConfig instanceConfig = HelixHelper.getInstanceConfig(_helixManager, instanceId); assertEquals(instanceConfig.getInstanceName(), instanceId); assertEquals(instanceConfig.getHostName(), "myHost"); - assertEquals(instanceConfig.getPort(), "1234"); + assertEquals(instanceConfig.getPort(), Integer.toString(controllerPort)); assertEquals(instanceConfig.getTags(), Set.of(CONTROLLER_INSTANCE)); stopController(); @@ -69,10 +71,11 @@ public void testHostnamePortOverride() @Test public void testInvalidInstanceId() throws Exception { + int controllerPort = NetUtils.findOpenPort(_nextControllerPort); _configOverride.clear(); _configOverride.put(CONFIG_OF_INSTANCE_ID, "myInstance"); _configOverride.put(CONTROLLER_HOST, "myHost"); - _configOverride.put(CONTROLLER_PORT, 1234); + _configOverride.put(CONTROLLER_PORT, controllerPort); startZk(); try { @@ -88,19 +91,20 @@ public void testInvalidInstanceId() @Test public void testDefaultInstanceId() throws Exception { + int controllerPort = NetUtils.findOpenPort(_nextControllerPort); _configOverride.clear(); _configOverride.put(CONTROLLER_HOST, "myHost"); - _configOverride.put(CONTROLLER_PORT, 1234); + _configOverride.put(CONTROLLER_PORT, controllerPort); startZk(); startController(); String instanceId = _controllerStarter.getInstanceId(); - assertEquals(instanceId, "Controller_myHost_1234"); + assertEquals(instanceId, "Controller_myHost_" + controllerPort); InstanceConfig instanceConfig = HelixHelper.getInstanceConfig(_helixManager, instanceId); assertEquals(instanceConfig.getInstanceName(), instanceId); assertEquals(instanceConfig.getHostName(), "myHost"); - assertEquals(instanceConfig.getPort(), "1234"); + assertEquals(instanceConfig.getPort(), Integer.toString(controllerPort)); assertEquals(instanceConfig.getTags(), Set.of(CONTROLLER_INSTANCE)); stopController(); diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java index 9c4317d32b27..6a45cf081bd9 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -121,18 +122,39 @@ public class ControllerTest { private static final Logger LOGGER = LoggerFactory.getLogger(ControllerTest.class); public static final String LOCAL_HOST = "localhost"; + // Use a random UUID rather than a timestamp so concurrent forks never share a data/temp dir + // (System.currentTimeMillis() collides when two forks initialize in the same millisecond). public static final String DEFAULT_DATA_DIR = new File(FileUtils.getTempDirectoryPath(), - "test-controller-data-dir" + System.currentTimeMillis()).getAbsolutePath(); + "test-controller-data-dir" + UUID.randomUUID()).getAbsolutePath(); public static final String DEFAULT_LOCAL_TEMP_DIR = new File(FileUtils.getTempDirectoryPath(), - "test-controller-local-temp-dir" + System.currentTimeMillis()).getAbsolutePath(); + "test-controller-local-temp-dir" + UUID.randomUUID()).getAbsolutePath(); public static final String BROKER_INSTANCE_ID_PREFIX = "Broker_localhost_"; public static final String SERVER_INSTANCE_ID_PREFIX = "Server_localhost_"; public static final String MINION_INSTANCE_ID_PREFIX = "Minion_localhost_"; public static final String TEST_PORT_BASE_PROPERTY = "pinot.test.port.base"; public static final String TEST_ZK_PORT_BASE_PROPERTY = "pinot.test.zk.port.base"; - private static final AtomicInteger NEXT_CONFIGURED_ZK_PORT = - new AtomicInteger(Integer.getInteger(TEST_ZK_PORT_BASE_PROPERTY, 0)); + /// Per-fork port offset so that concurrent surefire forks (forkCount > 1, reuseForks=false) + /// allocate disjoint port ranges. surefire injects a 1-based `surefire.forkNumber` into each + /// fork (so it is 1 even at forkCount=1, 1..N under parallel forks); it is 0 only outside a + /// surefire fork. The stride (5000) comfortably exceeds the ~3000-port span one ControllerTest + /// instance uses. Ports are still probed with findOpenPort, so the offset only separates the + /// per-fork starting points; no test depends on a literal base port. + private static final int FORK_PORT_OFFSET = forkNumber() * 5000; + + private static int forkNumber() { + try { + return Integer.parseInt(System.getProperty("surefire.forkNumber", "0")); + } catch (NumberFormatException e) { + return 0; + } + } + + // Offset only when an explicit ZK port base is configured; a base of 0 means "let ZkStarter + // pick the port" (which is already fork-aware), so it must stay 0 for forked runs too. + private static final int CONFIGURED_ZK_PORT_BASE = Integer.getInteger(TEST_ZK_PORT_BASE_PROPERTY, 0); + private static final AtomicInteger NEXT_CONFIGURED_ZK_PORT = new AtomicInteger( + CONFIGURED_ZK_PORT_BASE > 0 ? CONFIGURED_ZK_PORT_BASE + FORK_PORT_OFFSET : 0); // Default ControllerTest instance settings public static final int DEFAULT_MIN_NUM_REPLICAS = 2; @@ -150,7 +172,7 @@ public class ControllerTest { protected final String _clusterName = getClass().getSimpleName(); protected final List _fakeInstanceHelixManagers = new ArrayList<>(); - protected int _nextControllerPort = Integer.getInteger(TEST_PORT_BASE_PROPERTY, 20000); + protected int _nextControllerPort = Integer.getInteger(TEST_PORT_BASE_PROPERTY, 20000) + FORK_PORT_OFFSET; protected int _nextBrokerPort = _nextControllerPort + 1000; protected int _nextBrokerGrpcPort = _nextBrokerPort + 500; protected int _nextBrokerQueryRunnerPort = _nextBrokerGrpcPort + 250; diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java b/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java index 041fbbf3c49b..532b5dc2b0f6 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java @@ -21,6 +21,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import org.apache.pinot.common.metadata.segment.SegmentZKMetadata; import org.apache.pinot.common.partition.function.MurmurPartitionFunction; import org.apache.pinot.segment.spi.ColumnMetadata; @@ -34,6 +35,8 @@ public class SegmentMetadataMockUtils { + private static final AtomicLong UNIQUE_ID_GENERATOR = new AtomicLong(); + private SegmentMetadataMockUtils() { } @@ -60,22 +63,26 @@ public static SegmentMetadata mockSegmentMetadata(String tableName, String segme } public static SegmentMetadata mockSegmentMetadata(String tableName) { - String uniqueNumericString = Long.toString(System.nanoTime()); + String uniqueNumericString = nextUniqueNumericString(); return mockSegmentMetadata(tableName, tableName + uniqueNumericString, 100, uniqueNumericString); } public static SegmentMetadata mockSegmentMetadata(String tableName, long startTime, long endTime, TimeUnit timeUnit) { - String uniqueNumericString = Long.toString(System.nanoTime()); + String uniqueNumericString = nextUniqueNumericString(); return mockSegmentMetadata(tableName, tableName + uniqueNumericString, 100, uniqueNumericString, startTime, endTime, timeUnit); } public static SegmentMetadata mockSegmentMetadata(String tableName, String segmentName) { - String uniqueNumericString = Long.toString(System.nanoTime()); + String uniqueNumericString = nextUniqueNumericString(); return mockSegmentMetadata(tableName, segmentName, 100, uniqueNumericString); } + private static String nextUniqueNumericString() { + return Long.toString(UNIQUE_ID_GENERATOR.incrementAndGet()); + } + public static SegmentZKMetadata mockSegmentZKMetadata(String segmentName, long numTotalDocs) { SegmentZKMetadata segmentZKMetadata = Mockito.mock(SegmentZKMetadata.class); Mockito.when(segmentZKMetadata.getSegmentName()).thenReturn(segmentName); diff --git a/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json b/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json index a068eb371ee5..836d79b1e261 100644 --- a/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json +++ b/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json @@ -134,7 +134,7 @@ "select f from tableName where t between 1 and 1000": 2 }, "qps": 150, - "numMessagesPerSecInKafkaTopic":1000, + "numMessagesPerSecInKafkaTopic":100, "numRecordsPerPush":10000000, "tableType": "HYBRID", "latencySLA": 500, diff --git a/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_timeColumn.json b/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_timeColumn.json index e05a2f50ac6c..3d82e2489005 100644 --- a/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_timeColumn.json +++ b/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_timeColumn.json @@ -134,7 +134,7 @@ "select f from tableName where t between 1 and 1000": 2 }, "qps": 150, - "numMessagesPerSecInKafkaTopic":1000, + "numMessagesPerSecInKafkaTopic":100, "tableType": "HYBRID", "latencySLA": 500, "rulesToExecute": { diff --git a/pinot-core/src/test/java/org/apache/pinot/core/accounting/QueryMonitorConfigTest.java b/pinot-core/src/test/java/org/apache/pinot/core/accounting/QueryMonitorConfigTest.java index 85f9f02f165c..fcb7fb1021c2 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/accounting/QueryMonitorConfigTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/accounting/QueryMonitorConfigTest.java @@ -135,12 +135,13 @@ void testPanicLevelHeapUsageRatioConfigChange() { new PerQueryCPUMemResourceUsageAccountant(new PinotConfiguration(), "test", InstanceType.SERVER); assertEquals(accountant.getQueryMonitorConfig().getPanicLevel(), - Accounting.DEFAULT_PANIC_LEVEL_HEAP_USAGE_RATIO * accountant.getQueryMonitorConfig().getMaxHeapSize()); + (long) ((double) Accounting.DEFAULT_PANIC_LEVEL_HEAP_USAGE_RATIO + * accountant.getQueryMonitorConfig().getMaxHeapSize())); accountant.getWatcherTask() .onChange(Set.of(Accounting.COMMON_PREFIX + "." + Accounting.Keys.PANIC_LEVEL_HEAP_USAGE_RATIO), CLUSTER_CONFIGS); assertEquals(accountant.getQueryMonitorConfig().getPanicLevel(), - EXPECTED_PANIC_LEVEL * accountant.getQueryMonitorConfig().getMaxHeapSize()); + (long) (EXPECTED_PANIC_LEVEL * accountant.getQueryMonitorConfig().getMaxHeapSize())); } @Test @@ -149,12 +150,13 @@ void testCriticalLevelHeapUsageRatioConfigChange() { new PerQueryCPUMemResourceUsageAccountant(new PinotConfiguration(), "test", InstanceType.SERVER); assertEquals(accountant.getQueryMonitorConfig().getCriticalLevel(), - Accounting.DEFAULT_CRITICAL_LEVEL_HEAP_USAGE_RATIO * accountant.getQueryMonitorConfig().getMaxHeapSize()); + (long) ((double) Accounting.DEFAULT_CRITICAL_LEVEL_HEAP_USAGE_RATIO + * accountant.getQueryMonitorConfig().getMaxHeapSize())); accountant.getWatcherTask() .onChange(Set.of(Accounting.COMMON_PREFIX + "." + Accounting.Keys.CRITICAL_LEVEL_HEAP_USAGE_RATIO), CLUSTER_CONFIGS); assertEquals(accountant.getQueryMonitorConfig().getCriticalLevel(), - EXPECTED_CRITICAL_LEVEL * accountant.getQueryMonitorConfig().getMaxHeapSize()); + (long) (EXPECTED_CRITICAL_LEVEL * accountant.getQueryMonitorConfig().getMaxHeapSize())); } @Test @@ -163,12 +165,13 @@ void testAlarmingLevelHeapUsageRatioConfigChange() { new PerQueryCPUMemResourceUsageAccountant(new PinotConfiguration(), "test", InstanceType.SERVER); assertEquals(accountant.getQueryMonitorConfig().getAlarmingLevel(), - Accounting.DEFAULT_ALARMING_LEVEL_HEAP_USAGE_RATIO * accountant.getQueryMonitorConfig().getMaxHeapSize()); + (long) ((double) Accounting.DEFAULT_ALARMING_LEVEL_HEAP_USAGE_RATIO + * accountant.getQueryMonitorConfig().getMaxHeapSize())); accountant.getWatcherTask() .onChange(Set.of(Accounting.COMMON_PREFIX + "." + Accounting.Keys.ALARMING_LEVEL_HEAP_USAGE_RATIO), CLUSTER_CONFIGS); assertEquals(accountant.getQueryMonitorConfig().getAlarmingLevel(), - EXPECTED_ALARMING_LEVEL * accountant.getQueryMonitorConfig().getMaxHeapSize()); + (long) (EXPECTED_ALARMING_LEVEL * accountant.getQueryMonitorConfig().getMaxHeapSize())); } @Test diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java index d2b6aad8d306..5a5b7de8bdce 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -92,7 +93,8 @@ public class BaseTableDataManagerTest { - private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "BaseTableDataManagerTest"); + private static final File TEMP_DIR = + new File(FileUtils.getTempDirectory(), "BaseTableDataManagerTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "testTable"; private static final String OFFLINE_TABLE_NAME = TableNameBuilder.OFFLINE.tableNameWithType(RAW_TABLE_NAME); private static final File TABLE_DATA_DIR = new File(TEMP_DIR, OFFLINE_TABLE_NAME); diff --git a/pinot-core/src/test/java/org/apache/pinot/core/geospatial/transform/GeoFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/geospatial/transform/GeoFunctionTest.java index b00d3d94f2b9..b4f703bf6c58 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/geospatial/transform/GeoFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/geospatial/transform/GeoFunctionTest.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.function.BiConsumer; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.request.context.ExpressionContext; @@ -61,7 +62,8 @@ public abstract class GeoFunctionTest { private static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; - private static final String INDEX_DIR_PATH = FileUtils.getTempDirectoryPath() + File.separator + SEGMENT_NAME; + private static final String INDEX_DIR_PATH = + FileUtils.getTempDirectoryPath() + File.separator + SEGMENT_NAME + "-" + UUID.randomUUID(); private static final double DELTA = 0.00001; diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java index ccc4804e0ac3..fda7bc4c8bbe 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java @@ -123,7 +123,8 @@ public abstract class BaseTransformFunctionTest { /// index sitting on the column doesn't perturb the predicate evaluator's path selection. protected static final String INT_MV_DICT_RAW_INV_COLUMN = "intMVDictRawInv"; private static final String SEGMENT_NAME = "testSegment"; - private static final String INDEX_DIR_PATH = FileUtils.getTempDirectoryPath() + File.separator + SEGMENT_NAME; + private static final String INDEX_DIR_PATH = + FileUtils.getTempDirectoryPath() + File.separator + SEGMENT_NAME + "-" + UUID.randomUUID(); private static final Random RANDOM = new Random(); protected final int[] _intSVValues = new int[NUM_ROWS]; protected final long[] _longSVValues = new long[NUM_ROWS]; diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java index 26cd4f21587a..6536fd5f8b50 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; @@ -48,12 +49,14 @@ import org.apache.pinot.spi.utils.ReadMode; import org.apache.pinot.spi.utils.builder.TableConfigBuilder; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; public abstract class DistinctFromTransformFunctionTest { private static final String SEGMENT_NAME = "testSegment"; + private static final String INDEX_DIR_SUFFIX = "-" + UUID.randomUUID(); private static final String INT_SV_COLUMN = "intSV"; private static final String INT_SV_NULL_COLUMN = "intSV2"; private static final Random RANDOM = new Random(); @@ -65,6 +68,7 @@ public abstract class DistinctFromTransformFunctionTest { private final int[] _intSVValues = new int[NUM_ROWS]; private Map _dataSourceMap; + private IndexSegment _indexSegment; private ProjectionBlock _projectionBlock; DistinctFromTransformFunctionTest(boolean isDistinctFrom) { @@ -73,10 +77,10 @@ public abstract class DistinctFromTransformFunctionTest { } private static String getIndexDirPath(String segmentName) { - return FileUtils.getTempDirectoryPath() + File.separator + segmentName; + return FileUtils.getTempDirectoryPath() + File.separator + segmentName + INDEX_DIR_SUFFIX; } - private static Map getDataSourceMap(Schema schema, List rows, String segmentName) + private Map getDataSourceMap(Schema schema, List rows, String segmentName) throws Exception { TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(segmentName).setNullHandlingEnabled(true).build(); @@ -86,12 +90,11 @@ private static Map getDataSourceMap(Schema schema, List columnNames = indexSegment.getPhysicalColumnNames(); + _indexSegment = ImmutableSegmentLoader.load(new File(getIndexDirPath(segmentName), segmentName), ReadMode.heap); + Set columnNames = _indexSegment.getPhysicalColumnNames(); Map enableNullDataSourceMap = new HashMap<>(columnNames.size()); for (String columnName : columnNames) { - enableNullDataSourceMap.put(columnName, indexSegment.getDataSource(columnName)); + enableNullDataSourceMap.put(columnName, _indexSegment.getDataSource(columnName)); } return enableNullDataSourceMap; } @@ -143,6 +146,17 @@ public void setup() _projectionBlock = getProjectionBlock(_dataSourceMap); } + @AfterClass + public void tearDown() { + try { + if (_indexSegment != null) { + _indexSegment.destroy(); + } + } finally { + FileUtils.deleteQuietly(new File(getIndexDirPath(SEGMENT_NAME))); + } + } + protected void testTransformFunction(ExpressionContext expression, boolean[] expectedValues, ProjectionBlock projectionBlock, Map dataSourceMap) throws Exception { diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java index d681044951e8..9a41ec0100f6 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGeneratorTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.request.context.ExpressionContext; @@ -62,7 +63,8 @@ public class DictionaryBasedGroupKeyGeneratorTest { private static final String SEGMENT_NAME = "testSegment"; - private static final String INDEX_DIR_PATH = FileUtils.getTempDirectoryPath() + File.separator + SEGMENT_NAME; + private static final String INDEX_DIR_PATH = + FileUtils.getTempDirectoryPath() + File.separator + SEGMENT_NAME + "-" + UUID.randomUUID(); private static final int NUM_ROWS = 1000; private static final int UNIQUE_ROWS = 100; private static final int MAX_STEP_LENGTH = 1000; diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java index e5f6ad58983c..054f430c0749 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.commons.configuration2.PropertiesConfiguration; @@ -78,7 +79,8 @@ public class QueryExecutorExceptionsTest { private static final String AVRO_DATA_PATH = "data/simpleData200001.avro"; private static final String EMPTY_JSON_DATA_PATH = "data/test_empty_data.json"; private static final String QUERY_EXECUTOR_CONFIG_PATH = "conf/query-executor.properties"; - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "QueryExecutorTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "QueryExecutorTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "testTable"; private static final String OFFLINE_TABLE_NAME = TableNameBuilder.OFFLINE.tableNameWithType(RAW_TABLE_NAME); private static final int NUM_SEGMENTS_TO_GENERATE = 2; diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java index 318fd8e3fbf8..ecd4a223d886 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -96,7 +97,8 @@ public class QueryExecutorTest { private static final String AVRO_DATA_PATH = "data/sampleEatsData30k.avro"; private static final String EMPTY_JSON_DATA_PATH = "data/test_empty_data.json"; private static final String QUERY_EXECUTOR_CONFIG_PATH = "conf/query-executor.properties"; - private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "QueryExecutorTest"); + private static final File TEMP_DIR = + new File(FileUtils.getTempDirectory(), "QueryExecutorTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "sampleEatsData"; private static final String OFFLINE_TABLE_NAME = TableNameBuilder.OFFLINE.tableNameWithType(RAW_TABLE_NAME); private static final int NUM_SEGMENTS_TO_GENERATE = 2; diff --git a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java index cd4bb07bbd9e..6bba131d59d0 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.UUID; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.request.context.ExpressionContext; @@ -85,7 +86,8 @@ abstract class BaseStarTreeV2Test { private static final Random RANDOM = new Random(); - private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "BaseStarTreeV2Test"); + private static final File TEMP_DIR = + new File(FileUtils.getTempDirectory(), "BaseStarTreeV2Test-" + UUID.randomUUID()); protected static final String TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BaseFSTBasedRegexpLikeQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BaseFSTBasedRegexpLikeQueriesTest.java index 59eb2b4edae9..6b33fa8bb2d1 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseFSTBasedRegexpLikeQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseFSTBasedRegexpLikeQueriesTest.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.UUID; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.response.broker.BrokerResponseNative; @@ -62,7 +63,8 @@ public abstract class BaseFSTBasedRegexpLikeQueriesTest extends BaseQueriesTest { private static final File INDEX_DIR = - new File(FileUtils.getTempDirectory(), BaseFSTBasedRegexpLikeQueriesTest.class.getSimpleName()); + new File(FileUtils.getTempDirectory(), BaseFSTBasedRegexpLikeQueriesTest.class.getSimpleName() + "-" + + UUID.randomUUID()); private static final String TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; private static final String DOMAIN_NAMES_COL = "DOMAIN_NAMES"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BaseFunnelCountQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BaseFunnelCountQueriesTest.java index 9bed905c0791..603c1892007a 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseFunnelCountQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseFunnelCountQueriesTest.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.UUID; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; @@ -58,7 +59,7 @@ @SuppressWarnings("rawtypes") abstract public class BaseFunnelCountQueriesTest extends BaseQueriesTest { protected static final File INDEX_DIR = - new File(FileUtils.getTempDirectory(), "FunnelCountQueriesTest"); + new File(FileUtils.getTempDirectory(), "FunnelCountQueriesTest-" + UUID.randomUUID()); protected static final String RAW_TABLE_NAME = "testTable"; protected static final String SEGMENT_NAME = "testSegment"; protected static final Random RANDOM = new Random(); diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueQueriesTest.java index 268463cefce2..46910884d3e2 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueQueriesTest.java @@ -22,6 +22,7 @@ import java.net.URL; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader; import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl; @@ -61,7 +62,8 @@ /// - column10, METRIC, INT, 3960, F, F, F /// - daysSinceEpoch, TIME, INT, 1, T, F, F public abstract class BaseMultiValueQueriesTest extends BaseQueriesTest { - protected static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "MultiValueQueriesTest"); + protected static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "MultiValueQueriesTest-" + UUID.randomUUID()); protected static final String AVRO_DATA = "data" + File.separator + "test_data-mv.avro"; protected static final String RAW_TABLE_NAME = "testTable"; protected static final String SEGMENT_NAME = "testTable_1756015683_1756015683"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueRawQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueRawQueriesTest.java index b9ca42971aa8..7a6185ff9c50 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueRawQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseMultiValueRawQueriesTest.java @@ -22,6 +22,7 @@ import java.net.URL; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader; import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl; @@ -61,7 +62,8 @@ /// - column10, METRIC, INT, 3960, F, F, F /// - daysSinceEpoch, TIME, INT, 1, T, F, F public class BaseMultiValueRawQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "MultiValueRawQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "MultiValueRawQueriesTest-" + UUID.randomUUID()); private static final String AVRO_DATA = "data" + File.separator + "test_data-mv.avro"; protected static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testTable_1756015683_1756015683"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BaseSingleValueQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BaseSingleValueQueriesTest.java index 65c007b5c5d1..343d0d7c651d 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseSingleValueQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseSingleValueQueriesTest.java @@ -22,6 +22,7 @@ import java.net.URL; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader; import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl; @@ -62,7 +63,8 @@ /// - column18, METRIC, INT, 1440, F, T /// - daysSinceEpoch, TIME, INT, 2, T, F public abstract class BaseSingleValueQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "SingleValueQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "SingleValueQueriesTest-" + UUID.randomUUID()); private static final String AVRO_DATA = "data" + File.separator + "test_data-sv.avro"; protected static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testTable_126164076_167572854"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesScalabilityTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesScalabilityTest.java index 5e6a8aeab30e..391e490d4a5e 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesScalabilityTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesScalabilityTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.common.response.broker.ResultTable; @@ -49,7 +50,8 @@ /// Scalability Queries test for Gapfill queries. public class GapfillQueriesScalabilityTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "PostAggregationGapfillQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "PostAggregationGapfillQueriesTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "parkingData"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java index 3d7d1c839f53..3a42409b16a2 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.common.response.broker.ResultTable; @@ -51,7 +52,8 @@ /// Queries test for Gapfill queries. // TODO: Item 1. table alias for subquery in next PR public class GapfillQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "PostAggregationGapfillQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "PostAggregationGapfillQueriesTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "parkingData"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/HistogramQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/HistogramQueriesTest.java index b53eac4e97f7..aeb93a96c797 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/HistogramQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/HistogramQueriesTest.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.common.response.broker.ResultTable; @@ -59,7 +60,8 @@ /// Queries test for histogram queries. @SuppressWarnings({"rawtypes", "unchecked"}) public class HistogramQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "HistogramQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "HistogramQueriesTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/JsonIngestionFromAvroQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/JsonIngestionFromAvroQueriesTest.java index 0e78b88e4f4b..0555cd639648 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/JsonIngestionFromAvroQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/JsonIngestionFromAvroQueriesTest.java @@ -29,6 +29,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.avro.Schema; @@ -64,7 +65,8 @@ /// Test if ComplexType (RECORD, ARRAY, MAP, UNION, ENUM, and FIXED) field from an AVRO file can be ingested into a JSON /// column in a Pinot segment. public class JsonIngestionFromAvroQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "JsonIngestionFromAvroTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "JsonIngestionFromAvroTest-" + UUID.randomUUID()); private static final File AVRO_DATA_FILE = new File(INDEX_DIR, "JsonIngestionFromAvroTest.avro"); private static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/JsonMatchQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/JsonMatchQueriesTest.java index f28ab8fa4b59..5b6dba43401d 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/JsonMatchQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/JsonMatchQueriesTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader; @@ -54,7 +55,8 @@ /// Queries test for JSON_MATCH predicate. public class JsonMatchQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "JsonMatchQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "JsonMatchQueriesTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/JsonUnnestIngestionFromAvroQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/JsonUnnestIngestionFromAvroQueriesTest.java index 525840ddc5fe..9e400221f59c 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/JsonUnnestIngestionFromAvroQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/JsonUnnestIngestionFromAvroQueriesTest.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; import org.apache.avro.generic.GenericData; @@ -64,7 +65,8 @@ /// Test if ComplexType (RECORD, ARRAY, MAP, UNION, ENUM, and FIXED) field from an AVRO file can be ingested into a JSON /// column in a Pinot segment. public class JsonUnnestIngestionFromAvroQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "JsonIngestionFromAvroTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "JsonIngestionFromAvroTest-" + UUID.randomUUID()); private static final File AVRO_DATA_FILE = new File(INDEX_DIR, "JsonIngestionFromAvroTest.avro"); private static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/MultiValueRawQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/MultiValueRawQueriesTest.java index 8fbee3874668..4b9406725056 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/MultiValueRawQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/MultiValueRawQueriesTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pinot.common.response.broker.BrokerResponseNative; @@ -53,7 +54,8 @@ public class MultiValueRawQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "MultiValueRawQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "MultiValueRawQueriesTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME_1 = "testSegment1"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileKLLQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileKLLQueriesTest.java index 0aa7b2e697f6..7cc104ef5edc 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileKLLQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileKLLQueriesTest.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.datasketches.kll.KllDoublesSketch; import org.apache.pinot.common.response.broker.BrokerResponseNative; @@ -66,7 +67,8 @@ /// - Compares the results for PERCENTILE_KLL on double column and KLL column with results for PERCENTILE on /// double column public class PercentileKLLQueriesTest extends BaseQueriesTest { - protected static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "PercentileKllQueriesTest"); + protected static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "PercentileKllQueriesTest-" + UUID.randomUUID()); protected static final String TABLE_NAME = "testTable"; protected static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java index 97c76cdb04cb..497fc1b09616 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/PercentileTDigestQueriesTest.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.common.request.PinotQuery; import org.apache.pinot.common.response.broker.BrokerResponseNative; @@ -78,7 +79,8 @@ /// - Compares the results for PERCENTILE_TDIGEST on double column and TDigest column with results for PERCENTILE on /// double column public class PercentileTDigestQueriesTest extends BaseQueriesTest { - protected static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "PercentileTDigestQueriesTest"); + protected static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "PercentileTDigestQueriesTest-" + UUID.randomUUID()); protected static final String TABLE_NAME = "testTable"; protected static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/StatisticalQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/StatisticalQueriesTest.java index 5ded2ed8f8d7..75489ceabbae 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/StatisticalQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/StatisticalQueriesTest.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.List; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.commons.math3.stat.correlation.Covariance; import org.apache.commons.math3.stat.descriptive.moment.Kurtosis; @@ -66,7 +67,8 @@ /// Queries test for statistical queries (i.e Variance, Covariance, Standard Deviation etc) public class StatisticalQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "CovarianceQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "CovarianceQueriesTest-" + UUID.randomUUID()); private static final String RAW_TABLE_NAME = "testTable"; private static final String SEGMENT_NAME = "testSegment"; diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java index c1fb75b2237f..76ae66714411 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java @@ -30,6 +30,10 @@ import java.util.Map; import java.util.Objects; import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.lucene.analysis.Analyzer; @@ -85,7 +89,8 @@ /// The test table has a SKILLS column and QUERY_LOG column. Text index is created /// on each of these columns. public class TextSearchQueriesTest extends BaseQueriesTest { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "TextSearchQueriesTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "TextSearchQueriesTest-" + UUID.randomUUID()); protected static final String TABLE_NAME = "MyTable"; private static final String SEGMENT_NAME = "testSegment"; @@ -1591,31 +1596,29 @@ public void testLuceneRealtimeWithoutSearcherManager() public void testMultiThreadedLuceneRealtime() throws Exception { File indexFile = new File(INDEX_DIR.getPath() + "/realtime-test3.index"); - Directory indexDirectory = FSDirectory.open(indexFile.toPath()); - Analyzer analyzer = new CaseAwareStandardAnalyzer(); - // create and open a writer - IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); - indexWriterConfig.setRAMBufferSizeMB(500); - IndexWriter indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); - - // create an NRT index reader - SearcherManager searcherManager = new SearcherManager(indexWriter, false, false, null); - - // background thread to refresh NRT reader - ControlledRealTimeReopenThread controlledRealTimeReopenThread = - new ControlledRealTimeReopenThread(indexWriter, searcherManager, 0.01, 0.01); - controlledRealTimeReopenThread.start(); - - // start writer and reader - Thread writer = new Thread(new RealtimeWriter(indexWriter)); - Thread realtimeReader = new Thread(new RealtimeReader(searcherManager, analyzer)); - - writer.start(); - realtimeReader.start(); - - writer.join(); - realtimeReader.join(); - controlledRealTimeReopenThread.join(); + try (Directory indexDirectory = FSDirectory.open(indexFile.toPath()); + Analyzer analyzer = new CaseAwareStandardAnalyzer()) { + // create and open a writer + IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); + indexWriterConfig.setRAMBufferSizeMB(500); + try (IndexWriter indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); + SearcherManager searcherManager = new SearcherManager(indexWriter, false, false, null); + ControlledRealTimeReopenThread controlledRealTimeReopenThread = + new ControlledRealTimeReopenThread<>(indexWriter, searcherManager, 0.01, 0.01)) { + controlledRealTimeReopenThread.start(); + + // Start the writer and reader, and propagate worker failures back to the test thread. + ExecutorService executorService = Executors.newFixedThreadPool(2); + try { + Future writer = executorService.submit(new RealtimeWriter(indexWriter)); + Future realtimeReader = executorService.submit(new RealtimeReader(searcherManager, analyzer)); + writer.get(); + realtimeReader.get(); + } finally { + executorService.shutdownNow(); + } + } + } } private static class RealtimeWriter implements Runnable { @@ -1662,9 +1665,8 @@ public void run() { } finally { try { _indexWriter.commit(); - _indexWriter.close(); } catch (Exception e) { - throw new RuntimeException("Failed to commit/close the index writer"); + throw new RuntimeException("Failed to commit the index writer"); } } } @@ -1689,16 +1691,19 @@ public void run() { // in the index while (count < 1000) { IndexSearcher indexSearcher = _searcherManager.acquire(); - int hits = indexSearcher.search(query, Integer.MAX_VALUE).scoreDocs.length; - // TODO: see how we can make this more deterministic - if (count > 200) { - // we should see an increasing number of hits - assertTrue(hits > 0); - assertTrue(hits >= prevHits); + try { + int hits = indexSearcher.search(query, Integer.MAX_VALUE).scoreDocs.length; + // TODO: see how we can make this more deterministic + if (count > 200) { + // we should see an increasing number of hits + assertTrue(hits > 0); + assertTrue(hits >= prevHits); + } + count++; + prevHits = hits; + } finally { + _searcherManager.release(indexSearcher); } - count++; - prevHits = hits; - _searcherManager.release(indexSearcher); Thread.sleep(1); } } catch (Exception e) { diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/server/KafkaServerStartableTest.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/server/KafkaServerStartableTest.java index 5ac31b003770..42b5d48c6918 100644 --- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/server/KafkaServerStartableTest.java +++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/server/KafkaServerStartableTest.java @@ -65,11 +65,12 @@ public void testResolveKafkaBrokerListThrowsWhenNoBrokerAvailable() { } @Test - public void testStopReleasesBrokerPort() + public void testStopMakesBrokerUnavailable() throws Exception { int kafkaServerPort = NetUtils.findOpenPort(); + String brokerList = "localhost:" + kafkaServerPort; Properties serverProperties = new Properties(); - serverProperties.put("kafka.server.bootstrap.servers", "localhost:" + kafkaServerPort); + serverProperties.put("kafka.server.bootstrap.servers", brokerList); serverProperties.put("kafka.server.port", Integer.toString(kafkaServerPort)); serverProperties.put("kafka.server.broker.id", "0"); serverProperties.put("kafka.server.owner.name", getClass().getSimpleName()); @@ -79,11 +80,13 @@ public void testStopReleasesBrokerPort() kafkaServerStartable.init(serverProperties); kafkaServerStartable.start(); try { - Assert.assertFalse(NetUtils.available(kafkaServerPort), "Kafka port should be in use while broker is running"); + Assert.assertTrue(kafkaServerStartable.isKafkaAvailable(brokerList), + "Kafka broker should be reachable while running"); } finally { kafkaServerStartable.stop(); } - Assert.assertTrue(NetUtils.available(kafkaServerPort), "Kafka port should be released after broker stop"); + Assert.assertFalse(kafkaServerStartable.isKafkaAvailable(brokerList), + "Kafka broker should be unavailable after stop"); } private static final class TestableKafkaServerStartable extends KafkaServerStartable { diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarConsumerTest.java b/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarConsumerTest.java index f65cf67d3a07..42c91edcd07b 100644 --- a/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarConsumerTest.java +++ b/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarConsumerTest.java @@ -56,6 +56,9 @@ public class PulsarConsumerTest { private static final DockerImageName PULSAR_IMAGE = DockerImageName.parse("apachepulsar/pulsar:3.2.2"); + // The image defaults to 6 GiB, which can exhaust the Docker VM when unit tests use multiple forks. + private static final String PULSAR_MEMORY = "-Xms512m -Xmx1g -XX:MaxDirectMemorySize=1g"; + private static final Duration PULSAR_STARTUP_TIMEOUT = Duration.ofMinutes(10); public static final String TABLE_NAME_WITH_TYPE = "tableName_REALTIME"; public static final String TEST_TOPIC = "test-topic"; public static final String TEST_TOPIC_BATCH = "test-topic-batch"; @@ -75,7 +78,8 @@ public class PulsarConsumerTest { @BeforeClass public void setUp() throws Exception { - _pulsar = new PulsarContainer(PULSAR_IMAGE).withStartupTimeout(Duration.ofMinutes(5)); + _pulsar = new PulsarContainer(PULSAR_IMAGE).withEnv("PULSAR_MEM", PULSAR_MEMORY) + .withStartupTimeout(PULSAR_STARTUP_TIMEOUT); _pulsar.start(); try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(_pulsar.getHttpServiceUrl()).build()) { Topics topics = admin.topics(); @@ -88,10 +92,12 @@ public void setUp() } } - @AfterClass + @AfterClass(alwaysRun = true) public void tearDown() throws Exception { - _pulsar.stop(); + if (_pulsar != null) { + _pulsar.stop(); + } } public void publishRecords(PulsarClient client) diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java index f957d4509966..5614e7736422 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.PinotBuffersAfterMethodCheckRule; import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter; @@ -33,7 +34,8 @@ public class FixedBitIntReaderTest implements PinotBuffersAfterMethodCheckRule { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "FixedBitIntReaderTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "FixedBitIntReaderTest-" + UUID.randomUUID()); private static final int NUM_VALUES = 95; private static final Random RANDOM = new Random(); diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java index 4bfc2a3f67d5..f6afbf9a37c0 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java @@ -20,10 +20,12 @@ import java.io.File; import java.io.IOException; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; import org.apache.lucene.analysis.Analyzer; @@ -48,7 +50,8 @@ public class LuceneMutableTextIndexTest { private static final AtomicInteger SEGMENT_NAME_SUFFIX_COUNTER = new AtomicInteger(0); - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "LuceneMutableIndexTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "LuceneMutableIndexTest-" + UUID.randomUUID()); private static final String TEXT_COLUMN_NAME = "testColumnName"; private static final String CUSTOM_ANALYZER_FQCN = CustomAnalyzer.class.getName(); private static final String CUSTOM_QUERY_PARSER_FQCN = CustomQueryParser.class.getName(); @@ -64,14 +67,21 @@ public LuceneMutableTextIndexTest() { @BeforeMethod public void setUpMethod() { + // Give each test a fresh refresh queue and worker. Closing and immediately replacing indexes in the + // same queue can race the worker's empty-list exit and leave the replacement without a refresher. + RealtimeLuceneIndexRefreshManager.getInstance().reset(); _queryThreadContext = QueryThreadContext.openForSseTest(); } - @AfterMethod + @AfterMethod(alwaysRun = true) public void tearDownMethod() { - if (_queryThreadContext != null) { - _queryThreadContext.close(); - _queryThreadContext = null; + try { + closeCurrentIndex(); + } finally { + if (_queryThreadContext != null) { + _queryThreadContext.close(); + _queryThreadContext = null; + } } } @@ -179,6 +189,7 @@ private String[][] getRepeatedData() { private void configureIndex(String analyzerClass, String analyzerClassArgs, String analyzerClassArgTypes, String queryParserClass) { + closeCurrentIndex(); TextIndexConfigBuilder builder = new TextIndexConfigBuilder(); if (null != analyzerClass) { builder.withLuceneAnalyzerClass(analyzerClass); @@ -214,11 +225,34 @@ private void configureIndex(String analyzerClass, String analyzerClassArgs, Stri // ensure searches work after .commit() is called _realtimeLuceneTextIndex.commit(); - // sleep for index refresh - try { - Thread.sleep(100); - } catch (Exception e) { - // no-op + // Wait for the async NRT index refresh to make the committed documents searchable. A fixed + // sleep is flaky under CPU load (the refresh thread may not run in time), so poll until a + // sentinel query is visible, up to a generous timeout. + // + // The sentinel is the regex /.*house.*/ -> doc 1 ("...data warehouses"), which every test in + // this class also asserts and which matches under both the default StandardAnalyzer and the + // custom KeywordTokenizer (regex matches the single keyword-tokenized term). A term sentinel + // like "stream" would never match the keyword-tokenized custom-analyzer cases, making the + // barrier spin the full timeout for those tests. + awaitIndexRefreshed("/.*house.*/", ImmutableRoaringBitmap.bitmapOf(1)); + } + + private void awaitIndexRefreshed(String sentinelQuery, ImmutableRoaringBitmap expected) { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (true) { + if (expected.equals(_realtimeLuceneTextIndex.getDocIds(sentinelQuery))) { + return; + } + if (System.nanoTime() >= deadlineNanos) { + // Fall through and let the caller's assertions report the actual mismatch. + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } } } @@ -230,7 +264,18 @@ public void setUp() @AfterClass public void tearDown() { - _realtimeLuceneTextIndex.close(); + try { + closeCurrentIndex(); + } finally { + FileUtils.deleteQuietly(INDEX_DIR); + } + } + + private void closeCurrentIndex() { + if (_realtimeLuceneTextIndex != null) { + _realtimeLuceneTextIndex.close(); + _realtimeLuceneTextIndex = null; + } } @Test @@ -247,8 +292,8 @@ public void testGetSearchableDocCount() index.add(new String[]{"foo bar"}); index.add(new String[]{"baz qux"}); - // Force a searcher refresh — triggers the refresh listener which records the current doc count - index.getSearcherManager().maybeRefresh(); + // Block until the refresh attempt completes so the listener has recorded the current doc count + index.getSearcherManager().maybeRefreshBlocking(); assertEquals(index.getSearchableDocCount(), 3); } finally { @@ -258,6 +303,7 @@ public void testGetSearchableDocCount() @Test public void testQueries() { + configureIndex(null, null, null, null); TestUtils.waitForCondition(aVoid -> { try { return _realtimeLuceneTextIndex.getSearcherManager().isSearcherCurrent(); @@ -275,6 +321,7 @@ public void testQueries() { expectedExceptionsMessageRegExp = ".*TEXT_MATCH query interrupted while querying the consuming segment.*") public void testQueryCancellationIsSuccessful() throws InterruptedException, ExecutionException { + configureIndex(null, null, null, null); // Avoid early finalization by not using Executors.newSingleThreadExecutor (java <= 20, JDK-8145304) ExecutorService baseExecutor = Executors.newFixedThreadPool(1); // Wrap with contextAwareExecutorService to propagate QueryThreadContext to child threads diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java index eaec194da843..28a63602f7aa 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema.Field; import org.apache.avro.file.DataFileStream; @@ -76,7 +77,10 @@ public class DictionariesTest implements PinotBuffersAfterMethodCheckRule { private static final String AVRO_DATA = "data/test_sample_data.avro"; - private static final File INDEX_DIR = new File(DictionariesTest.class.toString()); + // Per-run unique dir so this test never shares an index directory with DictionaryOptimiserTest + // (which derived its path from the same class) when the two run concurrently in parallel forks. + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectoryPath(), DictionariesTest.class.getSimpleName() + "-" + UUID.randomUUID()); private static final Map> UNIQUE_ENTRIES = new HashMap<>(); private static File _segmentDirectory; diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionaryOptimiserTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionaryOptimiserTest.java index 7447f57ba1b5..e9df72b00ba6 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionaryOptimiserTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionaryOptimiserTest.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.avro.file.DataFileStream; @@ -60,7 +61,10 @@ public class DictionaryOptimiserTest implements PinotBuffersAfterMethodCheckRule private static final Logger LOGGER = LoggerFactory.getLogger(DictionaryOptimiserTest.class); private static final String AVRO_DATA = "data/mixed_cardinality_data.avro"; - private static final File INDEX_DIR = new File(DictionariesTest.class.toString()); + // Per-class unique dir so this test never shares an index directory with DictionariesTest (which + // used the same DictionariesTest.class-derived path) when the two run concurrently in parallel forks. + private static final File INDEX_DIR = new File(FileUtils.getTempDirectoryPath(), + DictionaryOptimiserTest.class.getSimpleName() + "-" + UUID.randomUUID()); private static File _segmentDirectory; diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java index eb18a2349c6e..2b35db720de8 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.PinotBuffersAfterMethodCheckRule; import org.apache.pinot.segment.local.io.writer.impl.FixedBitMVForwardIndexWriter; @@ -39,7 +40,8 @@ public class FixedBitMVForwardIndexTest implements PinotBuffersAfterMethodCheckRule { - private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "FixedBitMVForwardIndexTest"); + private static final File TEMP_DIR = + new File(FileUtils.getTempDirectory(), "FixedBitMVForwardIndexTest-" + UUID.randomUUID()); private static final File INDEX_FILE = new File(TEMP_DIR, "testColumn" + V1Constants.Indexes.UNSORTED_MV_FORWARD_INDEX_FILE_EXTENSION); private static final int NUM_DOCS = 100; diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitSVForwardIndexReaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitSVForwardIndexReaderTest.java index 26d76661d0c6..b114d2dc6cc9 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitSVForwardIndexReaderTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitSVForwardIndexReaderTest.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.PinotBuffersAfterMethodCheckRule; import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter; @@ -37,7 +38,8 @@ public class FixedBitSVForwardIndexReaderTest implements PinotBuffersAfterMethodCheckRule { - private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "FixedBitMVForwardIndexTest"); + private static final File TEMP_DIR = + new File(FileUtils.getTempDirectory(), "FixedBitSVForwardIndexReaderTest-" + UUID.randomUUID()); private static final File INDEX_FILE = new File(TEMP_DIR, "testColumn" + V1Constants.Indexes.UNSORTED_SV_FORWARD_INDEX_FILE_EXTENSION); private static final int NUM_DOCS = 100; diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java index 05234c1f2a86..c7272504a375 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java @@ -1093,7 +1093,15 @@ public void testV3CreateInvertedIndices() // Create inverted index the second time. checkInvertedIndexCreation(true); - assertEquals(Files.getLastModifiedTime(singleFileIndex.toPath()), newLastModifiedTime); + // The second (no-op) creation must not rewrite the file. Assert the mtime did not advance + // meaningfully rather than requiring exact equality: the 2s sleeps above guarantee a real + // rewrite would move the mtime by ~2000ms, whereas an untouched file can still report a + // sub-millisecond-to-millisecond delta between two getLastModifiedTime reads (filesystem + // timestamp granularity / metadata flush), which made exact equality flaky under CPU load. + long mtimeDeltaMs = + Files.getLastModifiedTime(singleFileIndex.toPath()).toMillis() - newLastModifiedTime.toMillis(); + assertTrue(Math.abs(mtimeDeltaMs) < 1000, + "columns.psf was rewritten by the no-op index recreation (mtime moved " + mtimeDeltaMs + " ms)"); assertEquals(singleFileIndex.length(), newFileSize); } diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java index 37b71514af8b..e421678cfe57 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Random; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.PinotBuffersAfterMethodCheckRule; import org.apache.pinot.segment.local.io.util.PinotDataBitSetV2; @@ -36,7 +37,8 @@ public class FixedBitSVForwardIndexReaderV2Test implements PinotBuffersAfterMethodCheckRule { - private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "FixedBitIntReaderTest"); + private static final File INDEX_DIR = + new File(FileUtils.getTempDirectory(), "FixedBitSVForwardIndexReaderV2Test-" + UUID.randomUUID()); private static final int NUM_VALUES = 99_999; private static final int NUM_DOC_IDS = PinotDataBitSetV2.MAX_DOC_PER_CALL; private static final Random RANDOM = new Random(); diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java index 87e9920b5029..6ae0d3757188 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java @@ -19,6 +19,7 @@ package org.apache.pinot.segment.local.segment.store; import java.io.File; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.pinot.segment.local.PinotBuffersAfterClassCheckRule; import org.apache.pinot.segment.spi.creator.SegmentVersion; @@ -35,7 +36,10 @@ public class SegmentLocalFSDirectoryTest implements PinotBuffersAfterClassCheckRule { - private static final File TEST_DIRECTORY = new File(SingleFileIndexDirectoryTest.class.toString()); + // Self-scoped unique dir (was derived from SingleFileIndexDirectoryTest.class) so parallel forks + // never share a directory. + private static final File TEST_DIRECTORY = new File(FileUtils.getTempDirectoryPath(), + SegmentLocalFSDirectoryTest.class.getSimpleName() + "-" + UUID.randomUUID()); private SegmentDirectory _segmentDirectory; private SegmentMetadataImpl _metadata; diff --git a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotDataBufferTestBase.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotDataBufferTestBase.java index 4e87fa86fe18..66d907dc9d3d 100644 --- a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotDataBufferTestBase.java +++ b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotDataBufferTestBase.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; import java.util.Random; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.commons.io.FileUtils; @@ -35,7 +36,8 @@ public class PinotDataBufferTestBase { protected static final Random RANDOM = new Random(); protected ExecutorService _executorService; - protected static final File TEMP_FILE = new File(FileUtils.getTempDirectory(), "PinotDataBufferTest"); + protected static final File TEMP_FILE = + new File(FileUtils.getTempDirectory(), "PinotDataBufferTest-" + UUID.randomUUID()); protected static final int FILE_OFFSET = 10; // Not page-aligned protected static final int BUFFER_SIZE = 10_000; // Not page-aligned protected static final int CHAR_ARRAY_LENGTH = BUFFER_SIZE / Character.BYTES; diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java index 0e4297f86d3e..200fcce5cb0c 100644 --- a/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java +++ b/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.InputStream; import java.net.URI; +import java.nio.file.Files; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -71,11 +72,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; public abstract class BaseResourceTest { - protected static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "BaseResourceTest"); protected static final String RAW_TABLE_NAME = "testTable"; protected static final String REALTIME_TABLE_NAME = TableNameBuilder.REALTIME.tableNameWithType(RAW_TABLE_NAME); protected static final String OFFLINE_TABLE_NAME = TableNameBuilder.OFFLINE.tableNameWithType(RAW_TABLE_NAME); @@ -89,6 +88,7 @@ public abstract class BaseResourceTest { protected final Map _tableDataManagerMap = new HashMap<>(); protected final List _realtimeIndexSegments = new ArrayList<>(); protected final List _offlineIndexSegments = new ArrayList<>(); + protected File _tempDir; protected File _avroFile; protected AdminApiApplication _adminApiApplication; protected WebTarget _webTarget; @@ -105,13 +105,12 @@ public void setUp() throws Exception { ServerMetrics.register(mock(ServerMetrics.class)); - FileUtils.deleteQuietly(TEMP_DIR); - assertTrue(TEMP_DIR.mkdirs()); - // Copy the Avro fixture out of the classpath into TEMP_DIR so it is always backed by a real file. + _tempDir = Files.createTempDirectory(getClass().getSimpleName() + "-").toFile(); + // Copy the Avro fixture out of the classpath into the temp directory so it is always backed by a real file. // The fixture may be served from a packaged test-jar when this base class is reused from another // module, in which case it cannot be opened as a plain File via the resource URL. String avroFileName = getAvroFileName(); - _avroFile = new File(TEMP_DIR, new File(avroFileName).getName()); + _avroFile = new File(_tempDir, new File(avroFileName).getName()); try (InputStream avroStream = getClass().getClassLoader().getResourceAsStream(avroFileName)) { assertNotNull(avroStream); FileUtils.copyInputStreamToFile(avroStream, _avroFile); @@ -128,7 +127,7 @@ public void setUp() when(_serverInstance.getServerMetrics()).thenReturn(mock(ServerMetrics.class)); when(_serverInstance.getInstanceDataManager()).thenReturn(instanceDataManager); when(_serverInstance.getInstanceDataManager().getSegmentFileDirectory()).thenReturn( - FileUtils.getTempDirectoryPath()); + _tempDir.getAbsolutePath()); // Create a single HelixManager mock with proper segment data HelixManager helixManager = mock(HelixManager.class); @@ -165,11 +164,12 @@ public void setUp() mock(ServerReloadJobStatusCache.class), serverConf); _adminApiApplication.start(List.of( - new ListenerConfig(CommonConstants.HTTP_PROTOCOL, "0.0.0.0", CommonConstants.Server.DEFAULT_ADMIN_API_PORT, + new ListenerConfig(CommonConstants.HTTP_PROTOCOL, "0.0.0.0", 0, CommonConstants.HTTP_PROTOCOL, new TlsConfig(), HttpServerThreadPoolConfig.defaultInstance()))); + int adminApiPort = _adminApiApplication.getHttpServer().getListeners().iterator().next().getPort(); _webTarget = ClientBuilder.newClient().target( - String.format("http://%s:%d", NetUtils.getHostAddress(), CommonConstants.Server.DEFAULT_ADMIN_API_PORT)); + String.format("http://%s:%d", NetUtils.getHostAddress(), adminApiPort)); } protected void configureServerConf(PinotConfiguration serverConf) { @@ -186,7 +186,7 @@ public void tearDown() { immutableSegment.offload(); immutableSegment.destroy(); } - FileUtils.deleteQuietly(TEMP_DIR); + FileUtils.deleteQuietly(_tempDir); } protected List setUpSegments(String tableNameWithType, int numSegments, @@ -208,7 +208,7 @@ protected ImmutableSegment setUpSegment(String tableNameWithType, String segment protected ImmutableSegment setUpSegment(String tableNameWithType, String segmentName, String segmentNamePostfix, List segments, boolean compressionStatsEnabled) throws Exception { - File tableDataDir = new File(TEMP_DIR, tableNameWithType); + File tableDataDir = new File(_tempDir, tableNameWithType); SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGeneratorConfigWithoutTimeColumn(_avroFile, tableDataDir, tableNameWithType); config.setSegmentName(segmentName); @@ -226,7 +226,7 @@ protected ImmutableSegment setUpSegment(String tableNameWithType, String segment protected void addTable(String tableNameWithType) { InstanceDataManagerConfig instanceDataManagerConfig = mock(InstanceDataManagerConfig.class); - when(instanceDataManagerConfig.getInstanceDataDir()).thenReturn(TEMP_DIR.getAbsolutePath()); + when(instanceDataManagerConfig.getInstanceDataDir()).thenReturn(_tempDir.getAbsolutePath()); when(instanceDataManagerConfig.getInstanceId()).thenReturn("Server_1_100.89.121.12"); TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); assertNotNull(tableType); diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java index fe912f3055c5..742614358f97 100644 --- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java +++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java @@ -471,7 +471,7 @@ private void downloadAndVerifySegmentContent(String tableNameWithType, IndexSegm Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); File segmentFile = response.readEntity(File.class); - File tempMetadataDir = new File(FileUtils.getTempDirectory(), "segment_metadata"); + File tempMetadataDir = new File(_tempDir, "segment_metadata"); FileUtils.forceMkdir(tempMetadataDir); // Extract metadata.properties @@ -750,7 +750,7 @@ public void testGetTableMetadataMixedDictRawCodec() .build(); // Segment 1: dictionary-encoded with tracked uncompressed value bytes. - File tableDataDir = new File(TEMP_DIR, mixedTableName); + File tableDataDir = new File(_tempDir, mixedTableName); TableConfig dictTableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(mixedTableName).build(); dictTableConfig.getIndexingConfig().setCompressionStatsEnabled(true); SegmentGeneratorConfig dictConfig = new SegmentGeneratorConfig(dictTableConfig, schema); diff --git a/pinot-spi/pom.xml b/pinot-spi/pom.xml index 34fda62848f4..11ac645deddf 100644 --- a/pinot-spi/pom.xml +++ b/pinot-spi/pom.xml @@ -193,7 +193,8 @@ org.apache.maven.plugins maven-surefire-plugin - 1 + + ${unit.test.fork.count} true diff --git a/pom.xml b/pom.xml index c03a30f031c0..f2ee4ebf33c0 100644 --- a/pom.xml +++ b/pom.xml @@ -139,9 +139,36 @@ true false + + 4g - -Xms4g -Xmx4g + -Xms${unit.test.fork.heap} -Xmx${unit.test.fork.heap} true + + 1 + + 0 + + warning @@ -557,7 +584,9 @@ false - -Xms4g -Xmx4g -Dlog4j2.configurationFile=log4j2.xml + + -Xms${unit.test.fork.heap} -Xmx${unit.test.fork.heap} -Dlog4j2.configurationFile=log4j2.xml @@ -578,6 +607,13 @@ org/apache/pinot/**/* + + ${project.build.directory}/jacoco${jacoco.exec.suffix}.exec @@ -2167,13 +2203,20 @@ org.apache.maven.plugins maven-surefire-plugin - 1 + + ${unit.test.fork.count} false + + ${unit.test.rerun.count} 3600 no + + $${surefire.forkNumber} false plain @@ -2513,8 +2556,32 @@ **/*IT.java - 1 + + ${unit.test.fork.count} false + + ${unit.test.rerun.count} + + + + usedefaultlisteners + false + + + + + $${surefire.forkNumber} +