run most JIT-sensitive performance tests at the very beginning to minimize other tests' influence

platform/platform-tests/testSrc/com/intellij/openapi/application/impl/ApplicationImplTest.java

@@ -29,10 +29,7 @@ import com.intellij.openapi.util.Disposer;
 import com.intellij.openapi.util.EmptyRunnable;
 import com.intellij.openapi.util.Ref;
 import com.intellij.openapi.util.ThrowableComputable;
-import com.intellij.testFramework.CpuUsageData;
-import com.intellij.testFramework.LightPlatformTestCase;
-import com.intellij.testFramework.LoggedErrorProcessor;
-import com.intellij.testFramework.PlatformTestUtil;
+import com.intellij.testFramework.*;
 import com.intellij.util.ArrayUtil;
 import com.intellij.util.ConcurrencyUtil;
 import com.intellij.util.ExceptionUtil;
@@ -52,6 +49,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+@JITSensitive
 public class ApplicationImplTest extends LightPlatformTestCase {
   @Override
   protected void setUp() throws Exception {

platform/platform-tests/testSrc/com/intellij/openapi/vfs/VfsUtilPerformanceTest.java

@@ -27,6 +27,7 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.newvfs.ManagingFS;
 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
 import com.intellij.testFramework.EdtTestUtil;
+import com.intellij.testFramework.JITSensitive;
 import com.intellij.testFramework.PlatformTestUtil;
 import com.intellij.testFramework.SkipSlowTestLocally;
 import com.intellij.testFramework.fixtures.BareTestFixtureTestCase;
@@ -48,6 +49,7 @@ import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.*;
 
+@JITSensitive
 @SkipSlowTestLocally
 public class VfsUtilPerformanceTest extends BareTestFixtureTestCase {
   @Rule public TempDirectory myTempDir = new TempDirectory();

platform/testFramework/src/com/intellij/TestCaseLoader.java

@@ -27,6 +27,7 @@ package com.intellij;
 import com.intellij.idea.Bombed;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.testFramework.JITSensitive;
 import com.intellij.testFramework.PlatformTestUtil;
 import com.intellij.testFramework.TestRunnerUtil;
 import com.intellij.util.containers.MultiMap;
@@ -198,31 +199,34 @@ public class TestCaseLoader {
     return Collections.emptyList();
   }
 
-  private int getRank(Class aClass) {
-    final String name = aClass.getName();
-    if (aClass == myFirstTestClass) return -1;
-    if (aClass == myLastTestClass) return myClassList.size() + ourRankList.size();
-    int i = ourRankList.indexOf(name);
+  private static int getRank(Class aClass) {
+    if (TestAll.isPerformanceTestsRun()) {
+      return moveToStart(aClass) ? 0 : 1;
+    }
+
+    int i = ourRankList.indexOf(aClass.getName());
     if (i != -1) {
       return i;
     }
     return ourRankList.size();
   }
 
+  private static boolean moveToStart(Class testClass) {
+    return testClass.getAnnotation(JITSensitive.class) != null;
+  }
+
   public List<Class> getClasses() {
     List<Class> result = new ArrayList<>(myClassList.size());
-    if (myFirstTestClass != null) {
-      result.add(myFirstTestClass);
-    }
     result.addAll(myClassList);
+    Collections.sort(result, Comparator.comparingInt(TestCaseLoader::getRank));
+    
+    if (myFirstTestClass != null) {
+      result.add(0, myFirstTestClass);
+    }
     if (myLastTestClass != null) {
       result.add(myLastTestClass);
     }
 
-    if (!ourRankList.isEmpty()) {
-      Collections.sort(result, (o1, o2) -> getRank(o1) - getRank(o2));
-    }
-
     return result;
   }
 

platform/testFramework/src/com/intellij/testFramework/JITSensitive.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.testFramework;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that annotated class contains performance tests that are so sensitive to just-in-time fluctuations that they
+ * need a specially controlled environment to run, with minimum effects from other tests. 
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface JITSensitive {
+}