junit: don't detect custom scope in IDE, leave it for ServiceLoader: if find custom engines, then try to start with junit 5

java/execution/impl/src/com/intellij/execution/junit/JUnitUtil.java

@@ -259,7 +259,7 @@ public class JUnitUtil {
       return aPackage != null && aPackage.getDirectories(scope).length > 0;
     };
 
-    return foundCondition.value(TEST5_PACKAGE_FQN) || foundCondition.value("org.junit.platform.engine");
+    return foundCondition.value(TEST5_PACKAGE_FQN);
   }
   
   public static boolean isTestAnnotated(final PsiMethod method) {

plugins/junit/src/com/intellij/execution/junit/TestObject.java

@@ -169,13 +169,18 @@ public abstract class TestObject extends JavaTestFrameworkRunnableState<JUnitCon
       File lib = new File(PathUtil.getJarPathForClass(MultipleFailuresError.class)).getParentFile();
       File[] files = lib.listFiles();
       if (files != null) {
+        JavaPsiFacade psiFacade = JavaPsiFacade.getInstance(project);
         for (File file : files) {
           String fileName = file.getName();
-          if (fileName.startsWith("junit-platform-") ||
-              fileName.startsWith("junit-jupiter-engine-") && JavaPsiFacade.getInstance(project).findClass(JUnitUtil.TEST5_ANNOTATION, globalSearchScope) != null) {
+          if (fileName.startsWith("junit-platform-launcher-") && !hasPackageWithDirectories(psiFacade, "org.junit.platform.launcher", globalSearchScope) ||
+
+              fileName.startsWith("junit-platform-") && !hasPackageWithDirectories(psiFacade, "org.junit.platform", globalSearchScope) ||
+
+              fileName.startsWith("junit-jupiter-engine-") && !hasPackageWithDirectories(psiFacade, "org.junit.jupiter.engine", globalSearchScope) &&
+                                                               hasPackageWithDirectories(psiFacade, JUnitUtil.TEST5_PACKAGE_FQN, globalSearchScope)) {
             classPath.add(file.getAbsolutePath());
           }
-          else if (fileName.startsWith("junit-vintage-engine-")) {
+          else if (fileName.startsWith("junit-vintage-engine-") && !hasPackageWithDirectories(psiFacade, "org.junit.vintage", globalSearchScope)) {
             try {
               JUnitUtil.getTestCaseClass(sourceScope);
               classPath.add(file.getAbsolutePath());
@@ -190,6 +195,13 @@ public abstract class TestObject extends JavaTestFrameworkRunnableState<JUnitCon
     return javaParameters;
   }
 
+  private static boolean hasPackageWithDirectories(JavaPsiFacade psiFacade,
+                                                   String packageQName,
+                                                   GlobalSearchScope globalSearchScope) {
+    PsiPackage aPackage = psiFacade.findPackage(packageQName);
+    return aPackage != null && aPackage.getDirectories(globalSearchScope).length > 0;
+  }
+
   public static boolean isJUnit5(@Nullable Module module, @Nullable SourceScope sourceScope, Project project) {
     return JUnitUtil.isJUnit5(getScopeForJUnit(module, sourceScope, project), project);
   }

plugins/junit_rt/src/com/intellij/rt/execution/junit/JUnit5EngineDetector.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.rt.execution.junit;
+
+import java.util.Iterator;
+import java.util.ServiceLoader;
+
+public class JUnit5EngineDetector {
+
+  public static boolean hasCustomEngine() {
+    try {
+      Iterator iterator = ServiceLoader.load(Class.forName("org.junit.platform.engine.TestEngine")).iterator();
+      while (iterator.hasNext()) {
+        Object engine = iterator.next();
+        String engineClassName = engine.getClass().getName();
+        if (!"org.junit.jupiter.engine.JupiterTestEngine".equals(engineClassName) &&
+            !"org.junit.vintage.engine.VintageTestEngine".equals(engineClassName)) {
+          return true;
+        }
+      }
+      return false;
+    }
+    catch (Throwable e) {
+      return false;
+    }
+  }
+}

plugins/junit_rt/src/com/intellij/rt/execution/junit/JUnitStarter.java

@@ -177,8 +177,13 @@ public class JUnitStarter {
 
   public static boolean isJUnit5Preferred() {
     final String useJUnit5 = System.getProperty(JUNIT5_KEY);
-    final Boolean boolValue = useJUnit5 == null ? null : Boolean.valueOf(useJUnit5);
-    return boolValue != null && boolValue.booleanValue();
+    if (useJUnit5 == null) {
+      return JUnit5EngineDetector.hasCustomEngine();
+    }
+    else {
+      final Boolean boolValue = Boolean.valueOf(useJUnit5);
+      return boolValue != null && boolValue.booleanValue();
+    }
   }
 
   public static boolean checkVersion(String[] args, PrintStream printStream) {