Don't depend on testFramework and make python-openapi not depend on lang

GitOrigin-RevId: c34a51bb984110d087fdddeff41c304f362ba4fe

platform/analysis-impl/intellij.platform.analysis.impl.iml

@@ -20,5 +20,6 @@
     <orderEntry type="library" name="kotlin-stdlib-jdk8" level="project" />
     <orderEntry type="module" module-name="intellij.platform.externalSystem" />
     <orderEntry type="module" module-name="intellij.platform.util.ui" />
+    <orderEntry type="library" name="Guava" level="project" />
   </component>
-</module>
+</module>

python/openapi/intellij.python.community.iml

@@ -8,10 +8,12 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="module" module-name="intellij.python.psi" exported="" />
-    <orderEntry type="module" module-name="intellij.platform.ide.impl" />
-    <orderEntry type="module" module-name="intellij.platform.lang" />
-    <orderEntry type="module" module-name="intellij.platform.lang.impl" />
+    <orderEntry type="module" module-name="intellij.platform.analysis" exported="" />
+    <orderEntry type="module" module-name="intellij.platform.analysis.impl" exported="" />
+    <orderEntry type="module" module-name="intellij.platform.ide" exported="" />
+
     <orderEntry type="library" name="Guava" level="project" />
     <orderEntry type="library" name="jsoup" level="project" />
+    <orderEntry type="library" name="kotlin-stdlib-jdk8" level="project" />
   </component>
-</module>
+</module>

python/openapi/src/com/jetbrains/python/module/PyModuleService.java

@@ -15,12 +15,10 @@
  */
 package com.jetbrains.python.module;
 
-import com.intellij.ide.util.projectWizard.ModuleBuilder;
 import com.intellij.openapi.components.ServiceManager;
 import com.intellij.openapi.module.Module;
 import com.intellij.openapi.projectRoots.Sdk;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.platform.DirectoryProjectGenerator;
 import com.intellij.util.Consumer;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -39,16 +37,6 @@ public abstract class PyModuleService {
     return ServiceManager.getService(PyModuleService.class);
   }
 
-  /**
-   * Creates a ModuleBuilder that creates a Python module and runs the specified DirectoryProjectGenerator to perform
-   * further initialization. The showGenerationSettings() method on the generator is not called, and the generateProject() method
-   * receives null as the 'settings' parameter.
-   *
-   * @param generator the generator to run for configuring the project
-   * @return the created module builder instance
-   */
-  public abstract ModuleBuilder createPythonModuleBuilder(DirectoryProjectGenerator generator);
-
 
   public boolean isFileIgnored(@NotNull VirtualFile file) {
     return false;

python/openapi/src/com/jetbrains/python/run/PyRunConfigurationFactory.java

@@ -1,8 +1,6 @@
 // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python.run;
 
-import com.intellij.execution.RunnerAndConfigurationSettings;
-import com.intellij.execution.configurations.ConfigurationFactory;
 import com.intellij.openapi.components.ServiceManager;
 import com.intellij.openapi.module.Module;
 
@@ -22,6 +20,4 @@ public abstract class PyRunConfigurationFactory {
    * @return the settings of the created run configuration.
    */
   public abstract PythonRunConfigurationParams createPythonScriptRunConfiguration(Module module, String scriptName);
-
-  public abstract RunnerAndConfigurationSettings createRunConfiguration(Module module, ConfigurationFactory factory);
 }

python/python-psi-impl/intellij.python.psi.impl.iml

@@ -17,11 +17,11 @@
     <orderEntry type="library" name="kotlin-stdlib-jdk8" level="project" />
     <orderEntry type="module" module-name="intellij.python.community" />
     <orderEntry type="library" name="jsoup" level="project" />
-    <orderEntry type="module" module-name="intellij.platform.testFramework" scope="TEST" />
     <orderEntry type="module" module-name="intellij.platform.indexing.impl" />
     <orderEntry type="library" name="StreamEx" level="project" />
     <orderEntry type="module" module-name="intellij.platform.projectModel.impl" />
     <orderEntry type="module" module-name="intellij.platform.analysis.impl" />
     <orderEntry type="module" module-name="intellij.platform.serviceContainer" />
+    <orderEntry type="library" name="JUnit4" level="project" scope="TEST"/>
   </component>
 </module>

python/python-psi-impl/test/com/intellij/openapi/application/impl/MockPsiApplication.kt

@@ -6,13 +6,14 @@ import com.intellij.mock.MockFileDocumentManagerImpl
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.application.*
 import com.intellij.openapi.application.ex.ApplicationEx
+import com.intellij.openapi.components.ComponentManager
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.impl.DocumentImpl
+import com.intellij.openapi.extensions.DefaultPluginDescriptor
 import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.*
 import com.intellij.serviceContainer.PlatformComponentManagerImpl
-import com.intellij.testFramework.registerServiceInstance
 import org.jetbrains.ide.PooledThreadExecutor
 import java.awt.Component
 import java.util.concurrent.Callable
@@ -230,4 +231,8 @@ class MockPsiApplication(parentDisposable: Disposable) : PlatformComponentManage
   override fun tryRunReadAction(action: Runnable): Boolean {
     TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
   }
-}
+}
+
+fun <T : Any> ComponentManager.registerServiceInstance(serviceInterface: Class<T>, instance: T) {
+  (this as PlatformComponentManagerImpl).registerServiceInstance(serviceInterface, instance, DefaultPluginDescriptor("test"))
+}

python/python-psi-impl/test/com/intellij/openapi/project/impl/MockPsiProject.kt

@@ -2,18 +2,13 @@ package com.intellij.openapi.project.impl
 
 import com.intellij.ide.plugins.ContainerDescriptor
 import com.intellij.ide.plugins.IdeaPluginDescriptorImpl
-import com.intellij.mock.MockApplication
-import com.intellij.mock.MockProject
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.application.Application
 import com.intellij.openapi.application.ApplicationManager
-import com.intellij.openapi.extensions.impl.ExtensionsAreaImpl
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.vfs.VirtualFile
-import com.intellij.serviceContainer.NonInjectable
 import com.intellij.serviceContainer.PlatformComponentManagerImpl
-import com.intellij.testFramework.registerComponentInstance
 
 class MockPsiProject(application: Application, parentDisposable: Disposable = application) : PlatformComponentManagerImpl(application), Project {
   init {
@@ -48,4 +43,4 @@ class MockPsiProject(application: Application, parentDisposable: Disposable = ap
   override fun isInitialized(): Boolean = true
 
   override fun isDefault(): Boolean = false
-}
+}

python/python-psi-impl/test/com/jetbrains/python/PythonMockSdk.java

@@ -33,6 +33,7 @@ import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.Comparator;
 
 /**
  * @author yole
@@ -72,43 +73,27 @@ public class PythonMockSdk {
     return sdk.clone();
   }
 
-  private static class PyMockSdkType extends SdkType {
+  private static class PyMockSdkType implements SdkTypeId {
 
     @NotNull
     private final String myVersionString;
+    private final String mySdkIdName;
 
     private PyMockSdkType(@NotNull String string) {
-      super(PyNames.PYTHON_SDK_ID_NAME);
+      mySdkIdName = PyNames.PYTHON_SDK_ID_NAME;
       myVersionString = string;
     }
 
-    @Nullable
-    @Override
-    public String suggestHomePath() {
-      return null;
-    }
-
-    @Override
-    public boolean isValidSdkHome(String path) {
-      return true;
-    }
-
     @NotNull
     @Override
-    public String suggestSdkName(@Nullable String currentSdkName, String sdkHome) {
-      return "Python";
+    public String getName() {
+      return mySdkIdName;
     }
 
     @Nullable
     @Override
-    public AdditionalDataConfigurable createAdditionalDataConfigurable(@NotNull SdkModel sdkModel, @NotNull SdkModificator sdkModificator) {
-      return null;
-    }
-
-    @NotNull
-    @Override
-    public String getPresentableName() {
-      return "Python";
+    public String getVersionString(@NotNull Sdk sdk) {
+      return myVersionString;
     }
 
     @Override
@@ -118,8 +103,19 @@ public class PythonMockSdk {
 
     @Nullable
     @Override
-    public String getVersionString(String sdkHome) {
-      return myVersionString;
+    public SdkAdditionalData loadAdditionalData(@NotNull Sdk currentSdk, @NotNull Element additional) {
+      return null;
+    }
+
+    @Override
+    public boolean isLocalSdk(@NotNull Sdk sdk) {
+      return false;
+    }
+
+    @NotNull
+    @Override
+    public Comparator<Sdk> versionComparator() {
+      return null;
     }
   }
 }

python/python-psi-impl/test/com/jetbrains/python/fixtures/PyPsiResolveTestCase.kt

@@ -1,16 +1,13 @@
 package com.jetbrains.python.fixtures
 
 import com.intellij.openapi.util.text.StringUtil
-import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.TestLocalVirtualFileSystem
 import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.psi.PsiManager
 import com.intellij.psi.PsiReference
-import com.intellij.testFramework.TestDataFile
 import com.jetbrains.python.PythonTestUtil
 import org.jetbrains.annotations.NonNls
-import java.io.File
 import java.io.IOException
 
 @NonNls
@@ -20,7 +17,7 @@ open class PyPsiResolveTestCase : PyPsiTestCase() {
 
   private val myFileSystem = TestLocalVirtualFileSystem()
 
-  protected fun configureByFile(@TestDataFile filePath: String): PsiReference? {
+  protected fun configureByFile(filePath: String): PsiReference? {
     val testDataRoot = myFileSystem.refreshAndFindFileByPath(PythonTestUtil.getTestDataPath())
 
     val file = testDataRoot!!.findFileByRelativePath(filePath)
@@ -41,4 +38,4 @@ open class PyPsiResolveTestCase : PyPsiTestCase() {
     val psiFile = PsiManager.getInstance(myProject).findFile(file)
     return psiFile?.findReferenceAt(offset)
   }
-}
+}

python/python-psi-impl/test/com/jetbrains/python/fixtures/PyPsiTestCase.kt

@@ -1,39 +1,48 @@
 package com.jetbrains.python.fixtures
 
 import com.intellij.core.CoreFileTypeRegistry
-import com.intellij.ide.plugins.IdeaPluginDescriptorImpl
 import com.intellij.ide.plugins.PluginManagerCore
-import com.intellij.idea.IdeaTestApplication
-import com.intellij.mock.MockApplication
-import com.intellij.mock.MockFileTypeManager
-import com.intellij.mock.MockProject
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.impl.MockPsiApplication
-import com.intellij.openapi.extensions.Extensions
-import com.intellij.openapi.extensions.impl.ExtensionsAreaImpl
-import com.intellij.openapi.fileTypes.FileTypeManager
 import com.intellij.openapi.fileTypes.FileTypeRegistry
 import com.intellij.openapi.project.impl.MockPsiProject
 import com.intellij.openapi.util.Getter
 import com.intellij.openapi.util.text.StringUtil
-import com.intellij.openapi.vfs.LocalFileSystem
-import com.intellij.openapi.vfs.VfsUtilCore
-import com.intellij.psi.PsiManager
-import com.intellij.psi.PsiReference
-import com.intellij.testFramework.PlatformTestUtil
-import com.intellij.testFramework.TestDataFile
 import com.intellij.util.PlatformUtils
 import com.jetbrains.python.PythonFileType
-import com.jetbrains.python.PythonTestUtil.getTestDataPath
 import junit.framework.TestCase
-import org.jetbrains.annotations.NonNls
-import java.io.File
-import java.io.IOException
+
+fun getTestName(name: String, lowercaseFirstLetter: Boolean): String {
+  var name = name
+  name = StringUtil.trimStart(name, "test")
+  return if (StringUtil.isEmpty(name)) "" else lowercaseFirstLetter(name, lowercaseFirstLetter)
+}
+
+fun lowercaseFirstLetter(name: String, lowercaseFirstLetter: Boolean): String {
+  var name = name
+  if (lowercaseFirstLetter && !isAllUppercaseName(name)) {
+    name = Character.toLowerCase(name[0]) + name.substring(1)
+  }
+  return name
+}
+
+fun isAllUppercaseName(name: String): Boolean {
+  var uppercaseChars = 0
+  for (i in 0 until name.length) {
+    if (Character.isLowerCase(name[i])) {
+      return false
+    }
+    if (Character.isUpperCase(name[i])) {
+      uppercaseChars++
+    }
+  }
+  return uppercaseChars >= 3
+}
 
 open class PyPsiTestCase : TestCase() {
   private val myParentDisposable: Disposable = TestDisposable()
-  private val myFileTypeManager = MockFileTypeManager(PythonFileType.INSTANCE)
+  private val myFileTypeManager = CoreFileTypeRegistry()
   private val myApplication: MockPsiApplication = MockPsiApplication(myParentDisposable)
     .also { ApplicationManager.setApplication(it, Getter { myFileTypeManager }, myParentDisposable) }
   protected val myProject: MockPsiProject = MockPsiProject(myApplication)
@@ -42,12 +51,13 @@ open class PyPsiTestCase : TestCase() {
     System.setProperty(PlatformUtils.PLATFORM_PREFIX_KEY, "PythonPsi")
     val loadedPlugins = PluginManagerCore.getLoadedPlugins(PyPsiTestCase::class.java.classLoader)
     myApplication.registerComponents(loadedPlugins)
-    myApplication.picoContainer.registerComponentInstance(FileTypeManager::class.java, myFileTypeManager)
+    myApplication.picoContainer.registerComponentInstance(FileTypeRegistry::class.java, myFileTypeManager)
     myProject.registerComponents(loadedPlugins)
+    myFileTypeManager.registerFileType(PythonFileType.INSTANCE, "py")
   }
 
   protected fun getTestName(lowercaseFirstLetter: Boolean): String =
-    name?.let { PlatformTestUtil.getTestName(it, lowercaseFirstLetter) } ?: ""
+    name?.let { getTestName(it, lowercaseFirstLetter) } ?: ""
 
   inner class TestDisposable: Disposable {
     @Volatile
@@ -62,4 +72,4 @@ open class PyPsiTestCase : TestCase() {
       return this@PyPsiTestCase.javaClass.name + if (StringUtil.isEmpty(testName)) "" else ".test$testName"
     }
   }
-}
+}

python/python-psi-impl/test/com/jetbrains/python/lexer/PyStringLiteralLexerTest.java

@@ -1,59 +0,0 @@
-/*
- * Copyright 2000-2013 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.jetbrains.python.lexer;
-
-import com.jetbrains.python.PyTokenTypes;
-import com.jetbrains.python.fixtures.PyLexerTestCase;
-
-/**
- * @author yole
- */
-public class PyStringLiteralLexerTest extends PyLexerTestCase {
-  public void testBackslashN() {  // PY-1313
-    PyLexerTestCase.doLexerTest("u\"\\N{LATIN SMALL LETTER B}\"", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
-                                "Py:SINGLE_QUOTED_UNICODE", "VALID_STRING_ESCAPE_TOKEN", "Py:SINGLE_QUOTED_UNICODE");
-  }
-
-  public void testRawBackslashN() {
-    PyLexerTestCase.doLexerTest("r'[\\w\\']'", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_STRING), true,
-                                "r'[\\w\\']'");
-  }
-
-  // PY-20921
-  public void testIllegalPrefixes() {
-    PyLexerTestCase.doLexerTest("ff'foo'", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), "Py:SINGLE_QUOTED_UNICODE");
-    PyLexerTestCase.doLexerTest("fff'foo'", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), "Py:SINGLE_QUOTED_UNICODE");
-    PyLexerTestCase.doLexerTest("rrr'''foo'''", new PyStringLiteralLexer(PyTokenTypes.TRIPLE_QUOTED_UNICODE), "Py:TRIPLE_QUOTED_UNICODE");
-  }
-
-  // PY-21399
-  public void testBackslashBreaksAnyEscapeSequence() {
-    PyLexerTestCase.doLexerTest("'\\u\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), true, "'", "\\u", "\\'", ")");
-    PyLexerTestCase.doLexerTest("'\\u\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
-                                "Py:SINGLE_QUOTED_UNICODE", "INVALID_UNICODE_ESCAPE_TOKEN", "VALID_STRING_ESCAPE_TOKEN",
-                                "Py:SINGLE_QUOTED_UNICODE");
-
-    PyLexerTestCase.doLexerTest("'\\x\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), true, "'", "\\x", "\\'", ")");
-    PyLexerTestCase.doLexerTest("'\\x\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
-                                "Py:SINGLE_QUOTED_UNICODE", "INVALID_UNICODE_ESCAPE_TOKEN", "VALID_STRING_ESCAPE_TOKEN",
-                                "Py:SINGLE_QUOTED_UNICODE");
-
-    PyLexerTestCase.doLexerTest("'\\N{F\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), true, "'", "\\N{F", "\\'", ")");
-    PyLexerTestCase.doLexerTest("'\\N{F\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
-                                "Py:SINGLE_QUOTED_UNICODE", "INVALID_UNICODE_ESCAPE_TOKEN", "VALID_STRING_ESCAPE_TOKEN",
-                                "Py:SINGLE_QUOTED_UNICODE");
-  }
-}

python/python-psi-impl/test/com/jetbrains/python/module/MockPyModuleService.kt

@@ -1,16 +1,13 @@
 package com.jetbrains.python.module
 
-import com.intellij.ide.util.projectWizard.ModuleBuilder
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.projectRoots.Sdk
-import com.intellij.platform.DirectoryProjectGenerator
 import com.jetbrains.python.PythonMockSdk
-import com.jetbrains.python.fixtures.PyTestCase
 
-class MockPyModuleService: PyModuleService() {
-  private val mySdk = PythonMockSdk.create(PyTestCase.PYTHON_3_MOCK_SDK)
+val PYTHON_3_MOCK_SDK = "3.7"
+
+class MockPyModuleService : PyModuleService() {
+  private val mySdk = PythonMockSdk.create(PYTHON_3_MOCK_SDK)
 
   override fun findPythonSdk(module: Module): Sdk? = mySdk
-
-  override fun createPythonModuleBuilder(generator: DirectoryProjectGenerator<*>?): ModuleBuilder = throw UnsupportedOperationException()
-}
+}

python/src/com/jetbrains/python/module/PyModuleServiceEx.java

@@ -0,0 +1,17 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.module;
+
+import com.intellij.ide.util.projectWizard.ModuleBuilder;
+import com.intellij.platform.DirectoryProjectGenerator;
+
+public abstract class PyModuleServiceEx extends PyModuleService {
+  /**
+   * Creates a ModuleBuilder that creates a Python module and runs the specified DirectoryProjectGenerator to perform
+   * further initialization. The showGenerationSettings() method on the generator is not called, and the generateProject() method
+   * receives null as the 'settings' parameter.
+   *
+   * @param generator the generator to run for configuring the project
+   * @return the created module builder instance
+   */
+  public abstract ModuleBuilder createPythonModuleBuilder(DirectoryProjectGenerator generator);
+}

python/src/com/jetbrains/python/module/PyModuleServiceImpl.java

@@ -34,7 +34,7 @@ import org.jetbrains.annotations.Nullable;
 /**
  * @author yole
  */
-public class PyModuleServiceImpl extends PyModuleService {
+public class PyModuleServiceImpl extends PyModuleServiceEx {
   @Override
   public ModuleBuilder createPythonModuleBuilder(DirectoryProjectGenerator generator) {
     return new PythonModuleBuilderBase(generator);

python/src/com/jetbrains/python/run/PyRunConfigurationFactoryEx.java

@@ -0,0 +1,10 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.run;
+
+import com.intellij.execution.RunnerAndConfigurationSettings;
+import com.intellij.execution.configurations.ConfigurationFactory;
+import com.intellij.openapi.module.Module;
+
+public abstract class PyRunConfigurationFactoryEx extends PyRunConfigurationFactory {
+  public abstract RunnerAndConfigurationSettings createRunConfiguration(Module module, ConfigurationFactory factory);
+}

python/src/com/jetbrains/python/run/PyRunConfigurationFactoryImpl.java

@@ -12,7 +12,7 @@ import org.jetbrains.annotations.NotNull;
 /**
  * @author yole
  */
-public class PyRunConfigurationFactoryImpl extends PyRunConfigurationFactory {
+public class PyRunConfigurationFactoryImpl extends PyRunConfigurationFactoryEx {
   @Override
   public PythonRunConfigurationParams createPythonScriptRunConfiguration(Module module, String scriptName) {
     RunnerAndConfigurationSettings settings = createRunConfiguration(module, PythonConfigurationType.getInstance().getFactory());

python/testSrc/com/jetbrains/python/PyStringLiteralLexerTest.java

@@ -0,0 +1,46 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python;
+
+import com.jetbrains.python.PyTokenTypes;
+import com.jetbrains.python.fixtures.PyLexerTestCase;
+import com.jetbrains.python.lexer.PyStringLiteralLexer;
+
+/**
+ * @author yole
+ */
+public class PyStringLiteralLexerTest extends PyLexerTestCase {
+  public void testBackslashN() {  // PY-1313
+    PyLexerTestCase.doLexerTest("u\"\\N{LATIN SMALL LETTER B}\"", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
+                                "Py:SINGLE_QUOTED_UNICODE", "VALID_STRING_ESCAPE_TOKEN", "Py:SINGLE_QUOTED_UNICODE");
+  }
+
+  public void testRawBackslashN() {
+    doLexerTest("r'[\\w\\']'", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_STRING), true,
+                "r'[\\w\\']'");
+  }
+
+  // PY-20921
+  public void testIllegalPrefixes() {
+    doLexerTest("ff'foo'", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), "Py:SINGLE_QUOTED_UNICODE");
+    doLexerTest("fff'foo'", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), "Py:SINGLE_QUOTED_UNICODE");
+    doLexerTest("rrr'''foo'''", new PyStringLiteralLexer(PyTokenTypes.TRIPLE_QUOTED_UNICODE), "Py:TRIPLE_QUOTED_UNICODE");
+  }
+
+  // PY-21399
+  public void testBackslashBreaksAnyEscapeSequence() {
+    doLexerTest("'\\u\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), true, "'", "\\u", "\\'", ")");
+    doLexerTest("'\\u\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
+                "Py:SINGLE_QUOTED_UNICODE", "INVALID_UNICODE_ESCAPE_TOKEN", "VALID_STRING_ESCAPE_TOKEN",
+                "Py:SINGLE_QUOTED_UNICODE");
+
+    doLexerTest("'\\x\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), true, "'", "\\x", "\\'", ")");
+    doLexerTest("'\\x\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
+                "Py:SINGLE_QUOTED_UNICODE", "INVALID_UNICODE_ESCAPE_TOKEN", "VALID_STRING_ESCAPE_TOKEN",
+                "Py:SINGLE_QUOTED_UNICODE");
+
+    doLexerTest("'\\N{F\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE), true, "'", "\\N{F", "\\'", ")");
+    doLexerTest("'\\N{F\\')", new PyStringLiteralLexer(PyTokenTypes.SINGLE_QUOTED_UNICODE),
+                "Py:SINGLE_QUOTED_UNICODE", "INVALID_UNICODE_ESCAPE_TOKEN", "VALID_STRING_ESCAPE_TOKEN",
+                "Py:SINGLE_QUOTED_UNICODE");
+  }
+}

python/testSrc/com/jetbrains/python/PythonCompletionTest.java

@@ -1,5 +1,5 @@
-// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.jetbrains.python.completion;
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python;
 
 import com.google.common.collect.Lists;
 import com.intellij.codeInsight.completion.CompletionType;

python/testSrc/com/jetbrains/python/PythonLexerTest.java

@@ -1,21 +1,8 @@
-/*
- * Copyright 2000-2016 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.jetbrains.python.lexer;
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python;
 
 import com.jetbrains.python.fixtures.PyLexerTestCase;
+import com.jetbrains.python.lexer.PythonIndentingLexer;
 
 /**
  * @author yole

python/testSrc/com/jetbrains/python/breadcrumbs/PyStringFormatParserTest.java

@@ -1,19 +1,5 @@
-/*
- * Copyright 2000-2013 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.jetbrains.python.parsing;
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.breadcrumbs;
 
 import com.intellij.openapi.util.TextRange;
 import com.intellij.testFramework.UsefulTestCase;
@@ -261,8 +247,8 @@ public class PyStringFormatParserTest extends TestCase {
     final List<Field> topLevelFields = result.getFields();
     UsefulTestCase.assertSize(2, topLevelFields);
     UsefulTestCase.assertSize(5, result.getAllFields());
-    UsefulTestCase.assertOrderedEquals(result.getAllFields().stream().map(Field::getDepth).toArray(), 1, 2, 3, 4, 1);
-    UsefulTestCase.assertOrderedEquals(result.getAllFields().stream().map(SubstitutionChunk::getAutoPosition).toArray(), 0, 1, 2, 3, 4);
+    assertOrderedEquals(result.getAllFields().stream().map(Field::getDepth).toArray(), 1, 2, 3, 4, 1);
+    assertOrderedEquals(result.getAllFields().stream().map(SubstitutionChunk::getAutoPosition).toArray(), 0, 1, 2, 3, 4);
   }
 
   public void testNewStyleAttrAndLookups() {
@@ -278,40 +264,40 @@ public class PyStringFormatParserTest extends TestCase {
 
     field = doParseAndGetFirstField("u'{foo.bar.baz}'");
     TestCase.assertEquals("foo", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), ".bar", ".baz");
+    assertOrderedEquals(field.getAttributesAndLookups(), ".bar", ".baz");
 
     field = doParseAndGetFirstField("u'{foo[bar][baz]}'");
     TestCase.assertEquals("foo", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[bar]", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[bar]", "[baz]");
 
     field = doParseAndGetFirstField("u'{foo.bar[baz]}'");
     TestCase.assertEquals("foo", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), ".bar", "[baz]");
 
     field = doParseAndGetFirstField("u'{foo.bar[baz}'");
     TestCase.assertEquals("foo", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), ".bar");
 
     field = doParseAndGetFirstField("u'{foo[{bar[baz]}'");
     TestCase.assertEquals("foo", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[{bar[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[{bar[baz]");
 
     field = doParseAndGetFirstField("u'{foo[{} {0} {bar.baz}]'");
     TestCase.assertEquals("foo", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[{} {0} {bar.baz}]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[{} {0} {bar.baz}]");
 
     field = doParseAndGetFirstField("u'{foo[bar]baz'");
     TestCase.assertEquals("foo", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[bar]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[bar]");
 
     field = doParseAndGetFirstField("'{0[foo][.!:][}]}'");
     TestCase.assertEquals("0", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", "[.!:]", "[}]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", "[.!:]", "[}]");
 
     field = doParseAndGetFirstField("'{.foo.bar}'");
     UsefulTestCase.assertEmpty(field.getFirstName());
     TestCase.assertEquals(TextRange.create(2, 2), field.getFirstNameRange());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), ".foo", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), ".foo", ".bar");
 
     field = doParseAndGetFirstField("'{}'");
     UsefulTestCase.assertEmpty(field.getFirstName());
@@ -344,68 +330,68 @@ public class PyStringFormatParserTest extends TestCase {
     UsefulTestCase.assertEmpty(field.getAttributesAndLookups());
 
     field = doParseAndGetFirstField("'{0[foo].bar[baz]}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     field = doParseAndGetFirstField("'{0[foo].bar[baz]'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     field = doParseAndGetFirstField("'{0[foo].bar[baz]");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     // do not recover unfinished lookups
     field = doParseAndGetFirstField("'{0[foo].bar[ba}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
 
     field = doParseAndGetFirstField("'{0[foo].bar[ba!}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
 
     field = doParseAndGetFirstField("'{0[foo].bar[ba:}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
 
     field = doParseAndGetFirstField("'{0[foo].bar[ba'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
 
     field = doParseAndGetFirstField("'{0[foo].bar[ba");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
 
     // do not recover illegal attributes
     field = doParseAndGetFirstField("'{0[foo].bar[baz]quux}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     field = doParseAndGetFirstField("'{0[foo].bar[baz]quux!}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     field = doParseAndGetFirstField("'{0[foo].bar[baz]quux:}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     field = doParseAndGetFirstField("'{0[foo].bar[baz]quux'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     field = doParseAndGetFirstField("'{0[foo].bar[baz]quux");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar", "[baz]");
 
     field = doParseAndGetFirstField("'{0..}'");
     TestCase.assertEquals("0", field.getFirstName());
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), ".", ".");
+    assertOrderedEquals(field.getAttributesAndLookups(), ".", ".");
 
     // recover attributes
     field = doParseAndGetFirstField("'{0[foo].}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".");
 
     field = doParseAndGetFirstField("'{0[foo].'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".");
 
     field = doParseAndGetFirstField("'{0[foo].");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".");
 
     field = doParseAndGetFirstField("'{0[foo].bar}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
 
     field = doParseAndGetFirstField("'{0[foo].bar'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
 
     field = doParseAndGetFirstField("'{0[foo].bar");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[foo]", ".bar");
   }
 
   public void testNewStyleAutoPosition() {
@@ -444,12 +430,12 @@ public class PyStringFormatParserTest extends TestCase {
 
   public void testNewStyleNamedUnicodeEscapeInLookup() {
     final Field field = doParseAndGetFirstField("'{foo[\\N{ESCAPE WITH ]}]}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), "[\\N{ESCAPE WITH ]}]");
+    assertOrderedEquals(field.getAttributesAndLookups(), "[\\N{ESCAPE WITH ]}]");
   }
 
   public void testNewStyleNamedUnicodeEscapeInAttribute() {
     final Field field = doParseAndGetFirstField("'{foo.b\\N{ESCAPE WITH [}.b\\N{ESCAPE WITH .}}'");
-    UsefulTestCase.assertOrderedEquals(field.getAttributesAndLookups(), ".b\\N{ESCAPE WITH [}", ".b\\N{ESCAPE WITH .}");
+    assertOrderedEquals(field.getAttributesAndLookups(), ".b\\N{ESCAPE WITH [}", ".b\\N{ESCAPE WITH .}");
   }
 
   public void testNewStyleUnclosedLookupEndsWithRightBrace() {

python/testSrc/com/jetbrains/python/breadcrumbs/PythonParsingTest.java

@@ -1,19 +1,5 @@
-/*
- * Copyright 2000-2016 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.jetbrains.python.parsing;
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.breadcrumbs;
 
 import com.intellij.lang.LanguageASTFactory;
 import com.intellij.psi.PsiFile;

python/testSrc/com/jetbrains/python/console/PythonInConsoleParsingTest.kt

@@ -3,7 +3,7 @@ package com.jetbrains.python.console
 
 import com.intellij.openapi.project.Project
 import com.jetbrains.python.PythonParserDefinition
-import com.jetbrains.python.parsing.PythonParsingTest
+import com.jetbrains.python.breadcrumbs.PythonParsingTest
 import com.jetbrains.python.parsing.console.PyConsoleParser
 import com.jetbrains.python.parsing.console.PythonConsoleData
 import com.jetbrains.python.parsing.console.PythonConsoleLexer