Merge remote-tracking branch 'origin/master'

build/scripts/common_tests.gant

@@ -71,7 +71,6 @@ target('run_tests': 'Run java tests') {
     [
       "idea.test.group",
       "idea.test.patterns",
-      "idea.fast.only",
       "idea.coverage.enabled.build",
       "teamcity.tests.recentlyFailedTests.file"
     ].each { pass(jvmArgs, it) }

build/scripts/tests.gant

@@ -32,13 +32,6 @@ else {
   ]
 }
 
-if (System.getProperty("os.name").toLowerCase().startsWith("mac")) {
-  String vendor = System.getProperty("java.vm.vendor")
-  if (vendor != null && vendor.toLowerCase().contains("apple")) {
-    args << "-d32"
-  }
-};
-
 args << "-Djna.nosys=true"
 
 setProperty("jvm_args", args)

java/java-tests/testSrc/com/intellij/codeInsight/daemon/impl/DaemonRespondToChangesTest.java

@@ -1688,6 +1688,8 @@ public class DaemonRespondToChangesTest extends DaemonAnalyzerTestCase {
     long e = System.currentTimeMillis();
     //System.out.println("Hi elapsed: "+(e-s));
 
+    //List<String> dumps = new ArrayList<>();
+
     final DaemonCodeAnalyzerImpl codeAnalyzer = (DaemonCodeAnalyzerImpl)DaemonCodeAnalyzer.getInstance(getProject());
     int N = Math.max(5, Timings.adjustAccordingToMySpeed(80, false));
     System.out.println("N = " + N);
@@ -1702,12 +1704,21 @@ public class DaemonRespondToChangesTest extends DaemonAnalyzerTestCase {
           // wait to engage all highlighting threads
           return;
         }
+        // uncomment to debug what's causing pauses
+        /*
+        AtomicBoolean finished = new AtomicBoolean();
+        AppExecutorUtil.getAppScheduledExecutorService().schedule(() -> {
+          if (!finished.get()) {
+            dumps.add(ThreadDumper.dumpThreadsToString());
+          }
+        }, 10, TimeUnit.MILLISECONDS);
+        */
         type(' ');
         long end = System.currentTimeMillis();
+        //finished.set(true);
         long interruptTime = end - now;
         interruptTimes[finalI] = interruptTime;
         assertTrue(codeAnalyzer.getUpdateProgress().isCanceled());
-        System.out.println(interruptTime);
         throw new ProcessCanceledException();
       };
       try {
@@ -1727,6 +1738,14 @@ public class DaemonRespondToChangesTest extends DaemonAnalyzerTestCase {
       //highlightErrors();
     }
 
+    System.out.println("Interrupt times: " + Arrays.toString(interruptTimes));
+
+    /*
+    for (String dump : dumps) {
+      System.out.println("\n\n-----------------------------\n\n" + dump);
+    }
+    */
+
     long mean = ArrayUtil.averageAmongMedians(interruptTimes, 3);
     long avg = Arrays.stream(interruptTimes).sum() / interruptTimes.length;
     long max = Arrays.stream(interruptTimes).max().getAsLong();

java/java-tests/testSrc/com/intellij/psi/resolve/ResolveClassTest.java

@@ -208,8 +208,9 @@ public class ResolveClassTest extends ResolveTestCase {
     assertInstanceOf(ref.resolve(), PsiClass.class);
   }
 
-
   public void testStaticImportInTheSameClassPerformance() throws Exception {
+    warmUpResolve();
+
     PsiReference ref = configure();
     ensureIndexUpToDate();
     long start = System.currentTimeMillis();
@@ -222,7 +223,16 @@ public class ResolveClassTest extends ResolveTestCase {
     getJavaFacade().findClass(CommonClassNames.JAVA_UTIL_LIST, GlobalSearchScope.allScope(myProject));
   }
 
+  private void warmUpResolve() {
+    PsiJavaCodeReferenceElement ref = JavaPsiFacade.getElementFactory(myProject).createReferenceFromText("java.util.List<String>", null);
+    JavaResolveResult result = ref.advancedResolve(false);
+    assertNotNull(result.getElement());
+    assertSize(1, result.getSubstitutor().getSubstitutionMap().keySet());
+  }
+
   public void testStaticImportNetworkPerformance() throws Exception {
+    warmUpResolve();
+
     PsiReference ref = configure();
     int count = 15;
 

java/openapi/src/com/intellij/util/xml/converters/values/GenericDomValueConvertersRegistry.java

@@ -20,7 +20,9 @@ import com.intellij.openapi.extensions.Extensions;
 import com.intellij.openapi.util.Comparing;
 import com.intellij.openapi.util.Condition;
 import com.intellij.openapi.util.Pair;
-import com.intellij.psi.PsiType;
+import com.intellij.psi.*;
+import com.intellij.psi.util.PsiTypesUtil;
+import com.intellij.psi.util.PsiUtil;
 import com.intellij.util.xml.Converter;
 import com.intellij.util.xml.GenericDomValue;
 import org.jetbrains.annotations.NotNull;
@@ -67,7 +69,16 @@ public class GenericDomValueConvertersRegistry {
   }
 
   public void registerClassValueConverters() {
-    registerConverter(ClassValueConverter.getClassValueConverter(), Class.class);
+    registerConverter(ClassValueConverter.getClassValueConverter(), pair -> {
+      PsiType psiType = pair.getFirst();
+      if (psiType instanceof PsiClassType) {
+        PsiClass resolve = ((PsiClassType)psiType).resolve();
+        if (resolve != null) {
+          return (CommonClassNames.JAVA_LANG_CLASS.equals(resolve.getQualifiedName()));
+        }
+      }
+      return false;
+    });
     registerConverter(ClassArrayConverter.getClassArrayConverter(), Class[].class);
   }
 
@@ -122,5 +133,4 @@ public class GenericDomValueConvertersRegistry {
     final String name = type.getCanonicalName();
     registerConverter(provider, pair -> pair.first != null && Comparing.equal(name, pair.first.getCanonicalText()));
   }
-
 }

platform/analysis-impl/src/com/intellij/codeInsight/daemon/impl/DaemonCodeAnalyzerEx.java

@@ -21,9 +21,9 @@ import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.editor.Document;
 import com.intellij.openapi.editor.ex.MarkupModelEx;
-import com.intellij.openapi.editor.ex.RangeHighlighterEx;
 import com.intellij.openapi.editor.impl.DocumentMarkupModel;
 import com.intellij.openapi.progress.ProgressIndicator;
+import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.Project;
 import com.intellij.psi.PsiFile;
 import com.intellij.util.CommonProcessors;
@@ -51,6 +51,7 @@ public abstract class DaemonCodeAnalyzerEx extends DaemonCodeAnalyzer {
     final SeverityRegistrar severityRegistrar = SeverityRegistrar.getSeverityRegistrar(project);
     MarkupModelEx model = (MarkupModelEx)DocumentMarkupModel.forDocument(document, project, true);
     return model.processRangeHighlightersOverlappingWith(startOffset, endOffset, marker -> {
+      ProgressManager.checkCanceled();
       Object tt = marker.getErrorStripeTooltip();
       if (!(tt instanceof HighlightInfo)) return true;
       HighlightInfo info = (HighlightInfo)tt;

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledJreManager.groovy

@@ -93,9 +93,9 @@ class BundledJreManager {
       return null
     }
     buildContext.messages.block("Extract $archive.name JRE") {
-      String destination = "$targetDir/jre32"
-      if (osDirName == "win" && arch == JvmArchitecture.x64) {
-        destination = "$targetDir/jre64"
+      String destination = "$targetDir/jre64"
+      if (osDirName == "win" && arch == JvmArchitecture.x32) {
+        destination = "$targetDir/jre32"
       }
       buildContext.messages.progress("Extracting JRE from '$archive.name' archive")
       if (SystemInfo.isWindows) {

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/CompilationContextImpl.groovy

@@ -130,7 +130,7 @@ class CompilationContextImpl implements CompilationContext {
       projectBuilder.targetFolder = classesOutput
     }
     else {
-      def outputDir = JpsPathUtil.urlToFile(JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(project).outputUrl)
+      def outputDir = getProjectOutputDirectory()
       if (!outputDir.exists()) {
         messages.error("$BuildOptions.USE_COMPILED_CLASSES_PROPERTY is enabled, but the project output directory $outputDir.absolutePath doesn't exist")
       }
@@ -141,6 +141,10 @@ class CompilationContextImpl implements CompilationContext {
     cleanOutput(outputDirectoriesToKeep)
   }
 
+  File getProjectOutputDirectory() {
+    JpsPathUtil.urlToFile(JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(project).outputUrl)
+  }
+
   void cleanOutput(List<String> outputDirectoriesToKeep) {
     messages.block("Clean output") {
       def outputPath = paths.buildOutputRoot

platform/core-impl/src/com/intellij/psi/stubs/LightStubBuilder.java

@@ -20,6 +20,7 @@ import com.intellij.openapi.diagnostic.LogUtil;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.fileTypes.LanguageFileType;
+import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.StubBuilder;
 import com.intellij.psi.impl.source.PsiFileImpl;
@@ -92,6 +93,8 @@ public class LightStubBuilder implements StubBuilder {
 
     nextElement:
     while (element != null) {
+      ProgressManager.checkCanceled();
+
       final StubElement stub = createStub(tree, element, parentStub);
       boolean hasStub = stub != parentStub || parent == null;
       if (hasStub && !immediateParentStubbed) {

platform/platform-impl/src/com/intellij/ide/ui/laf/intellij/MacComboBoxBorder.java

@@ -23,6 +23,7 @@ import com.intellij.util.ui.JBUI;
 import javax.swing.*;
 import java.awt.*;
 import java.awt.geom.Area;
+import java.awt.geom.Path2D;
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RoundRectangle2D;
 
@@ -37,6 +38,7 @@ public class MacComboBoxBorder extends MacIntelliJTextBorder {
 
     try {
       g2.translate(x, y);
+      g2.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
 
       Area area = new Area(new Rectangle2D.Double(0, 0, width, height));
       area.subtract(getButtonBounds(c));
@@ -46,11 +48,16 @@ public class MacComboBoxBorder extends MacIntelliJTextBorder {
 
       if (c instanceof JComboBox) {
         JComboBox comboBox = (JComboBox)c;
-        Color color = UIManager.getColor(comboBox.isEnabled() ? "ComboBox.background" : "ComboBox.disabledBackground");
-        RectanglePainter.paint(g2, JBUI.scale(3), JBUI.scale(3),
-                               c.getWidth() - JBUI.scale(6),
-                               c.getHeight() - JBUI.scale(6),
-                               arc, color, null);
+        g2.setColor(UIManager.getColor(comboBox.isEnabled() ? "ComboBox.background" : "ComboBox.disabledBackground"));
+        Path2D path = new Path2D.Double(Path2D.WIND_EVEN_ODD);
+        path.moveTo(JBUI.scale(8), JBUI.scale(3));
+        path.lineTo(JBUI.scale(8), c.getHeight() - JBUI.scale(3));
+        path.lineTo(JBUI.scale(3) + arc, c.getHeight() - JBUI.scale(3));
+        path.quadTo(JBUI.scale(3), c.getHeight() - JBUI.scale(3), JBUI.scale(3), c.getHeight() - JBUI.scale(3) - arc);
+        path.lineTo(JBUI.scale(3), JBUI.scale(3) + arc);
+        path.quadTo(JBUI.scale(3), JBUI.scale(3), JBUI.scale(3) + arc, JBUI.scale(3));
+        path.lineTo(JBUI.scale(8), JBUI.scale(3));
+        g2.fill(path);
       }
 
       RectanglePainter.paint(g2, JBUI.scale(3), JBUI.scale(3),

platform/platform-impl/src/com/intellij/ui/ComboboxSpeedSearch.java

@@ -56,6 +56,6 @@ public class ComboboxSpeedSearch extends SpeedSearchBase<JComboBox> {
   }
 
   protected String getElementText(Object element) {
-    return element.toString();
+    return element == null ? null : element.toString();
   }
 }

platform/testFramework/src/com/intellij/testFramework/LightPlatformTestCase.java

@@ -44,7 +44,6 @@ import com.intellij.openapi.editor.Document;
 import com.intellij.openapi.editor.Editor;
 import com.intellij.openapi.editor.EditorFactory;
 import com.intellij.openapi.editor.impl.EditorFactoryImpl;
-import com.intellij.openapi.editor.impl.EditorImpl;
 import com.intellij.openapi.fileEditor.FileDocumentManager;
 import com.intellij.openapi.fileEditor.impl.FileDocumentManagerImpl;
 import com.intellij.openapi.fileTypes.FileType;
@@ -87,13 +86,13 @@ import com.intellij.psi.impl.PsiDocumentManagerImpl;
 import com.intellij.psi.impl.PsiManagerImpl;
 import com.intellij.psi.impl.source.tree.injected.InjectedLanguageManagerImpl;
 import com.intellij.psi.templateLanguages.TemplateDataLanguageMappings;
-import com.intellij.util.ref.GCUtil;
 import com.intellij.util.IncorrectOperationException;
 import com.intellij.util.LocalTimeCounter;
 import com.intellij.util.ReflectionUtil;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.UnindexedFilesUpdater;
 import com.intellij.util.messages.MessageBusConnection;
+import com.intellij.util.ref.GCUtil;
 import com.intellij.util.ui.UIUtil;
 import junit.framework.AssertionFailedError;
 import junit.framework.TestCase;
@@ -266,8 +265,11 @@ public abstract class LightPlatformTestCase extends UsefulTestCase implements Da
   protected void setUp() throws Exception {
     EdtTestUtil.runInEdtAndWait(() -> {
       super.setUp();
-      initApplication();
       ApplicationInfoImpl.setInStressTest(isStressTest());
+      if (isPerformanceTest()) {
+        Timings.getStatistics();
+      }
+      initApplication();
 
       ourApplication.setDataProvider(this);
       LightProjectDescriptor descriptor = getProjectDescriptor();

platform/testFramework/src/com/intellij/testFramework/UsefulTestCase.java

@@ -130,6 +130,9 @@ public abstract class UsefulTestCase extends TestCase {
     }
     boolean isStressTest = isStressTest();
     ApplicationInfoImpl.setInStressTest(isStressTest);
+    if (isPerformanceTest()) {
+      Timings.getStatistics();
+    }
     // turn off Disposer debugging for performance tests
     Disposer.setDebugMode(!isStressTest);
   }

python/educational-core/course-creator/src/com/jetbrains/edu/coursecreator/CCSubtaskEditorNotificationProvider.java

@@ -233,6 +233,7 @@ public class CCSubtaskEditorNotificationProvider extends EditorNotifications.Pro
         taskFile.setTask(task);
       }
       list.set(i, task);
+      renameFiles(task.getTaskDir(myProject), myProject, -2);
     }
 
     private void updateInfoIndexes() {

python/educational-core/course-creator/src/com/jetbrains/edu/coursecreator/CCUtils.java

@@ -269,6 +269,9 @@ public class CCUtils {
     presentation.setEnabledAndVisible(project != null && isCourseCreator(project));
   }
 
+  /**
+   * @param fromIndex -1 if task converted to TaskWithSubtasks, -2 if task converted from TaskWithSubtasks
+   */
   public static void renameFiles(VirtualFile taskDir, Project project, int fromIndex) {
     ApplicationManager.getApplication().runWriteAction(() -> {
       Map<VirtualFile, String> newNames = new HashMap<>();
@@ -283,7 +286,7 @@ public class CCUtils {
             index = "0";
           }
           else { // remove subtask
-            index = subtaskIndex == 1 ? "" : Integer.toString(subtaskIndex - 1);
+            index = fromIndex == -2 ? "" : Integer.toString(subtaskIndex - 1);
           }
           String fileName = virtualFile.getName();
           String nameWithoutExtension = FileUtil.getNameWithoutExtension(fileName);

python/src/com/jetbrains/python/codeInsight/typing/PyTypingTypeProvider.java

@@ -118,6 +118,7 @@ public class PyTypingTypeProvider extends PyTypeProviderBase {
     return null;
   }
 
+  @Override
   @Nullable
   public Ref<PyType> getParameterType(@NotNull PyNamedParameter param, @NotNull PyFunction func, @NotNull TypeEvalContext context) {
     final Ref<PyType> typeFromAnnotation = getParameterTypeFromAnnotation(param, context);
@@ -784,6 +785,16 @@ public class PyTypingTypeProvider extends PyTypeProviderBase {
             continue;
           }
         }
+        if (isBuiltinPathLike(element)) {
+          // see https://github.com/python/typeshed/commit/41561f11c7b06368aebe512acf69d8010662266d
+          // or comment in typeshed/stdlib/3/builtins.pyi near _PathLike class
+          final QualifiedName osPathLikeQName = QualifiedName.fromComponents("os", PyNames.PATH_LIKE);
+          final PsiElement osPathLike = PyResolveImportUtil.resolveTopLevelMember(osPathLikeQName, PyResolveImportUtil.fromFoothold(element));
+          if (osPathLike != null) {
+            elements.add(osPathLike);
+            continue;
+          }
+        }
         if (element != null) {
           elements.add(element);
         }
@@ -792,6 +803,12 @@ public class PyTypingTypeProvider extends PyTypeProviderBase {
     return !elements.isEmpty() ? elements : Collections.singletonList(expression);
   }
 
+  private static boolean isBuiltinPathLike(@Nullable PsiElement element) {
+    return element instanceof PyClass &&
+           PyBuiltinCache.getInstance(element).isBuiltin(element) &&
+           ("_" + PyNames.PATH_LIKE).equals(((PyClass)element).getName());
+  }
+
   @NotNull
   private static Collection<String> resolveToQualifiedNames(@NotNull PyExpression expression, @NotNull TypeEvalContext context) {
     final Set<String> names = Sets.newLinkedHashSet();

python/src/com/jetbrains/python/testing/universalTests/PyUniversalNoseTest.kt

@@ -61,7 +61,7 @@ class PyUniversalNoseTestConfiguration(project: Project, factory: PyUniversalNos
   override fun isFrameworkInstalled() = VFSTestFrameworkListener.getInstance().isNoseTestInstalled(sdk)
 
   //https://github.com/nose-devs/nose/issues/1042
-  override fun treatFoldersAsPackages(anchor: PsiElement) = true
+  override fun packageOnlyIfInitPy(anchor: PsiElement) = true
 }
 
 object PyUniversalNoseTestFactory : PyUniversalTestFactory<PyUniversalNoseTestConfiguration>() {

python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt

@@ -529,7 +529,7 @@ abstract class PyUniversalTestConfiguration(project: Project,
   /**
    * When checking if configuration is ok we need to know if folders could be packages: i.e. if foo.bar requires init.py in foo to work
    */
-  open fun treatFoldersAsPackages(anchor: PsiElement) = (!LanguageLevel.forElement(anchor).isPy3K)
+  open fun packageOnlyIfInitPy(anchor: PsiElement) = (!LanguageLevel.forElement(anchor).isPy3K)
 }
 
 private fun isTestFile(file: PyFile): Boolean {
@@ -595,7 +595,7 @@ object PyUniversalTestsConfigurationProducer : AbstractPythonTestConfigurationPr
   private fun getTargetForConfig(configuration: PyUniversalTestConfiguration,
                                  baseElement: PsiElement, fixConfiguration: Boolean = false): ConfigurationTarget? {
 
-    val setRelative = (fixConfiguration && configuration.treatFoldersAsPackages(baseElement))
+    val setRelative = (fixConfiguration && configuration.packageOnlyIfInitPy(baseElement))
 
     var element = baseElement
     // Go up until we reach top of the file

python/src/com/jetbrains/python/testing/universalTests/PyUniversalUnitTest.kt

@@ -24,6 +24,7 @@ import com.intellij.execution.runners.ExecutionEnvironment
 import com.intellij.openapi.options.SettingsEditor
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.psi.PsiElement
 import com.jetbrains.python.PythonHelper
 import com.jetbrains.python.testing.PythonTestConfigurationsModel
 
@@ -74,6 +75,9 @@ class PyUniversalUnitTestConfiguration(project: Project, factory: PyUniversalUni
   }
 
   override fun isFrameworkInstalled() = true //Unittest is always available
+
+  // See loader.py:     is_not_importable = not os.path.isfile(os.path.join(start_dir, '__init__.py'))
+  override fun packageOnlyIfInitPy(anchor: PsiElement) = true
 }
 
 object PyUniversalUnitTestFactory : PyUniversalTestFactory<PyUniversalUnitTestConfiguration>() {

python/testData/inspections/PyTypeCheckerInspection/PathLikePassedToStdlibFunctions/a.py

@@ -28,11 +28,11 @@ class B:
 
 b = B()
 
-open(<warning descr="Expected type 'Union[str, bytes, int, PathLike]', got 'B' instead">b</warning>)
+open(<warning descr="Unexpected type(s):(B)Possible types:(Union[str, bytes, int])(Union[str, bytes, int, PathLike])">b</warning>)
 
-os.fspath(<warning descr="Expected type 'Union[str, bytes, int, PathLike]', got 'B' instead">b</warning>)
-os.fsencode(<warning descr="Expected type 'Union[str, bytes, int, PathLike]', got 'B' instead">b</warning>)
-os.fsdecode(<warning descr="Expected type 'Union[str, bytes, int, PathLike]', got 'B' instead">b</warning>)
+os.fspath(b)  # TODO fail after enabling pyi-stubs for `os` module
+os.fsencode(b)  # TODO fail after enabling pyi-stubs for `os` module
+os.fsdecode(b)  # TODO fail after enabling pyi-stubs for `os` module
 
 Path(<warning descr="Expected type 'Union[str, bytes, PathLike]', got 'B' instead">b</warning>)
 PurePath(<warning descr="Expected type 'Union[str, bytes, PathLike]', got 'B' instead">b</warning>)

python/testData/inspections/PyTypeCheckerInspection/PathLikePassedToStdlibFunctions/os.py

@@ -2,6 +2,20 @@ import abc
 import posixpath as path
 
 
+def _fscodec():
+    pass
+
+
+fsencode, fsdecode = _fscodec()
+
+
+def _fspath(path):
+    pass
+
+
+fspath = _fspath
+
+
 class PathLike(abc.ABC):
 
     """Abstract base class for implementing the file system path protocol."""