Try and reduce the vm crash on fork

Use the drop_caches and split unit tests

Ready for the review

Author: luocooong

.github/workflows/ci.yml

@@ -46,22 +46,20 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
               ${{ runner.os }}-maven-
-      # Caches MySQL directory used for JDBC storage plugin tests
-      - name: Cache MySQL
-        uses: actions/cache@v2
-        with:
-          path: ~/.embedmysql
-          key: ${{ runner.os }}-mysql
-      - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v2
-        with:
-          distribution: 'adopt'
-          java-version: ${{ matrix.java }}
       - name: Build and test
         # The total GitHub Actions memory is 7000Mb. But GitHub CI requires some memory for the container to perform tests
         run: |
           MAVEN_OPTS="-XX:+UseG1GC"
-          mvn install --batch-mode --no-transfer-progress -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 # -X -V for debugging
+          sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches' && \
+          mvn install --batch-mode --no-transfer-progress \
+          -DexcludedGroups=org.apache.drill.categories.EasyOutOfMemory \
+          -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
+      - name: Test Specific Categories # EasyOutOfMemory
+        run: |
+          sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches' && \
+          mvn test -pl org.apache.drill.exec:drill-java-exec \
+          -Dgroups=org.apache.drill.categories.EasyOutOfMemory \
+          -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
 
   checkstyle_protobuf:
     name: Run checkstyle and generate protobufs

.travis.yml

@@ -73,7 +73,7 @@ install:
   # For protobuf phase: builds Drill project, performs license checkstyle goal and regenerates Java and C++ Protobuf files
   - |
     if [ $PHASE = "tests" ]; then \
-      mvn install --batch-mode --no-transfer-progress -DforkCount=1 -DdirectMemoryMb=$DIRECTMEMORYMB \
+      mvn install --batch-mode --no-transfer-progress -DdirectMemoryMb=$DIRECTMEMORYMB \
         -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest,org.apache.drill.categories.SecurityTest"; \
     elif [ $PHASE = "build_checkstyle_protobuf" ]; then \
       MAVEN_OPTS="-Xms1G -Xmx1G" mvn install --no-transfer-progress -Drat.skip=false -Dlicense.skip=false --batch-mode -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true && \

common/src/test/java/org/apache/drill/categories/EasyOutOfMemory.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.categories;
+
+/**
+ * Split the unit test for EasyOutOfMemory categories.<br/>
+ * Note: If you use this category to mark the test class,
+ * the CI will use a new clean JVM to test them.
+ *
+ */
+public interface EasyOutOfMemory extends FlakyTest {
+
+}

common/src/test/java/org/apache/drill/categories/package-info.java

@@ -25,6 +25,8 @@
  *   <li>{@link org.apache.drill.categories.SlowTest}: These tests run slowly, and are disabled by default.</li>
  *   <li>{@link org.apache.drill.categories.UnlikelyTest}: These tests represent corner cases, specific bug fixes, or tests for pieces of code that are unlikely to be changed.
  *   These are disabled by default</li>
+ *   <li>{@link org.apache.drill.categories.FlakyTest}: A category for tests that intermittently fail.</li>
+ *   <li>{@link org.apache.drill.categories.EasyOutOfMemory}: Inherited class FlakyTest and allow the CI tool uses a new JVM to test the unit.</li>
  * </ul>
  * </p>
  * <p>

exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java

@@ -31,6 +31,7 @@
 import org.apache.drill.exec.record.RecordBatchLoader;
 import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.util.Text;
+import org.apache.drill.categories.EasyOutOfMemory;
 import org.apache.drill.categories.OperatorTest;
 import org.apache.drill.categories.PlannerTest;
 import org.apache.drill.categories.SqlFunctionTest;
@@ -68,7 +69,7 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-@Category({SqlFunctionTest.class, OperatorTest.class, PlannerTest.class})
+@Category({ SqlFunctionTest.class, OperatorTest.class, PlannerTest.class, EasyOutOfMemory.class })
 public class TestAggregateFunctions extends ClusterTest {
 
   @Rule

exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java

@@ -52,9 +52,6 @@
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
-import org.junit.jupiter.api.condition.DisabledOnOs;
-import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
-import org.junit.jupiter.api.condition.OS;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -1003,8 +1000,6 @@ public void testTPCHReadWriteGzip() throws Exception {
   // Only attempt this test on Linux / amd64 because com.rdblue.brotli-codec
   // only bundles natives for Mac and Linux on AMD64.  See PARQUET-1975.
   @Test
-  @EnabledIfSystemProperty(named = "os.arch", matches = "(amd64|x86_64)")
-  @DisabledOnOs({ OS.WINDOWS })
   public void testTPCHReadWriteBrotli() throws Exception {
     try {
       client.alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "brotli");

exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java

@@ -117,7 +117,7 @@ public class TestDrillbitResilience extends ClusterTest {
   private static final int NUM_RUNS = 3;
   private static final int PROBLEMATIC_TEST_NUM_RUNS = 3;
   private static final int TIMEOUT = 15;
-  private final static Level CURRENT_LOG_LEVEL = Level.DEBUG;
+  private final static Level CURRENT_LOG_LEVEL = Level.INFO;
 
   /**
    * Note: Counting sys.memory executes a fragment on every drillbit. This is a better check in comparison to

pom.xml

@@ -116,7 +116,7 @@
     <codemodel.version>2.6</codemodel.version>
     <joda.version>2.10.5</joda.version>
     <javax.el.version>3.0.0</javax.el.version>
-    <surefire.version>3.0.0-M5</surefire.version>
+    <surefire.version>3.0.0-M7</surefire.version>
     <jna.version>5.8.0</jna.version>
     <commons.compress.version>1.21</commons.compress.version>
     <hikari.version>4.0.3</hikari.version>