diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt new file mode 100644 index 000000000000..d6487e106577 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt @@ -0,0 +1,93 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.psi.stubs + +import com.google.common.hash.HashCode +import com.google.common.hash.Hashing +import com.intellij.openapi.extensions.ExtensionPointName +import com.intellij.openapi.fileTypes.FileTypeExtension +import com.intellij.util.indexing.FileContent +import com.intellij.util.io.DataExternalizer +import com.intellij.util.io.DataInputOutputUtil +import com.intellij.util.io.KeyDescriptor +import java.io.DataInput +import java.io.DataOutput +import java.nio.charset.Charset + +/** + * @author traff + */ + +val EP_NAME = "com.intellij.filetype.prebuiltStubsProvider" + +class PrebuiltStubsProviders : FileTypeExtension(EP_NAME) { + companion object { + val EXTENSION_POINT_NAME = ExtensionPointName.create(EP_NAME) + val instance = PrebuiltStubsProviders() + } +} + +interface PrebuiltStubsProvider { + fun findStub(fileContent: FileContent): Stub? +} + +class FileContentHashing { + private val hashing = Hashing.sha256() + + fun hashString(text: CharSequence, charset: Charset): HashCode { + return hashing.hashString(text, charset) + + } + + fun hashString(fileContent: FileContent): HashCode { + return hashString(fileContent.contentAsText, fileContent.file.charset) + } +} + + +class HashCodeDescriptor : HashCodeExternalizers(), KeyDescriptor { + override fun getHashCode(value: HashCode): Int = value.hashCode() + + + override fun isEqual(val1: HashCode, val2: HashCode): Boolean = val1 == val2 + + companion object { + val instance = HashCodeDescriptor() + } +} + +open class HashCodeExternalizers : DataExternalizer { + override fun save(out: DataOutput, value: HashCode) { + val bytes = value.asBytes() + DataInputOutputUtil.writeINT(out, bytes.size) + out.write(bytes, 0, bytes.size) + } + + override fun read(`in`: DataInput): HashCode { + val len = DataInputOutputUtil.readINT(`in`) + val bytes = ByteArray(len) + `in`.readFully(bytes) + return HashCode.fromBytes(bytes) + } +} + +class StubTreeExternalizer : DataExternalizer { + override fun save(out: DataOutput, value: SerializedStubTree) { + value.write(out) + } + + override fun read(`in`: DataInput) = SerializedStubTree(`in`) +} \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/SerializationManagerImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/SerializationManagerImpl.java index 700aed9d95d0..8ebdf25116df 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/SerializationManagerImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/SerializationManagerImpl.java @@ -37,12 +37,17 @@ public class SerializationManagerImpl extends SerializationManagerEx implements private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.SerializationManagerImpl"); private final AtomicBoolean myNameStorageCrashed = new AtomicBoolean(false); - private final File myFile = new File(PathManager.getIndexRoot(), "rep.names"); + private final File myFile; private final AtomicBoolean myShutdownPerformed = new AtomicBoolean(false); private AbstractStringEnumerator myNameStorage; private StubSerializationHelper myStubSerializationHelper; public SerializationManagerImpl() { + this(new File(PathManager.getIndexRoot(), "rep.names")); + } + + public SerializationManagerImpl(@NotNull File nameStorageFile) { + myFile = nameStorageFile; myFile.getParentFile().mkdirs(); try { // we need to cache last id -> String mappings due to StringRefs and stubs indexing that initially creates stubs (doing enumerate on String) diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java b/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java index 8893bfe4bafc..ba951b4ad684 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java @@ -54,7 +54,8 @@ public class SerializedStubTree { myLength = myBytes.length; myByteContentLength = DataInputOutputUtil.readLONG(in); myCharContentLength = DataInputOutputUtil.readINT(in); - } else { + } + else { myBytes = CompressionUtil.readCompressed(in); myLength = myBytes.length; myByteContentLength = in.readLong(); @@ -79,6 +80,11 @@ public class SerializedStubTree { // willIndexStub is one time optimization hint, once can safely pass false @NotNull public Stub getStub(boolean willIndexStub) throws SerializerNotFoundException { + return getStub(willIndexStub, SerializationManagerEx.getInstanceEx()); + } + + @NotNull + public Stub getStub(boolean willIndexStub, @NotNull SerializationManagerEx serializationManager) throws SerializerNotFoundException { Stub stubElement = myStubElement; if (stubElement != null) { // not null myStubElement means we just built SerializedStubTree for indexing, @@ -86,7 +92,7 @@ public class SerializedStubTree { myStubElement = null; if (willIndexStub) return stubElement; } - return SerializationManagerEx.getInstanceEx().deserialize(new UnsyncByteArrayInputStream(myBytes)); + return serializationManager.deserialize(new UnsyncByteArrayInputStream(myBytes)); } public boolean contentLengthMatches(long byteContentLength, int charContentLength) { @@ -111,7 +117,7 @@ public class SerializedStubTree { final byte[] thisBytes = myBytes; final byte[] thatBytes = thatTree.myBytes; - for (int i=0; i< length; i++) { + for (int i = 0; i < length; i++) { if (thisBytes[i] != thatBytes[i]) { return false; } @@ -121,8 +127,9 @@ public class SerializedStubTree { } public int hashCode() { - if (myBytes == null) - return 0; + if (myBytes == null) { + return 0; + } int result = 1; for (int i = 0; i < myLength; i++) { @@ -131,5 +138,4 @@ public class SerializedStubTree { return result; } - } \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 2a814de3b2d7..3724a6a94a17 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -25,6 +25,7 @@ import com.intellij.openapi.fileTypes.LanguageFileType; import com.intellij.openapi.progress.ProcessCanceledException; import com.intellij.openapi.util.ThrowableComputable; import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; +import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.newvfs.FileAttribute; import com.intellij.openapi.vfs.newvfs.NewVirtualFile; @@ -209,7 +210,20 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi }; ApplicationManager.getApplication().runReadAction(() -> { - final Stub rootStub = StubTreeBuilder.buildStubTree(inputData); + Stub rootStub = null; + + if (Registry.is("use.prebuilt.stubs")) { + final PrebuiltStubsProvider prebuiltStubsProvider = + PrebuiltStubsProviders.Companion.getInstance().forFileType(inputData.getFileType()); + if (prebuiltStubsProvider != null) { + rootStub = prebuiltStubsProvider.findStub(inputData); + } + } + + if (rootStub == null) { + rootStub = StubTreeBuilder.buildStubTree(inputData); + } + if (rootStub == null) return; VirtualFile file = inputData.getFile(); diff --git a/platform/platform-resources/src/META-INF/LangExtensionPoints.xml b/platform/platform-resources/src/META-INF/LangExtensionPoints.xml index a2c41b8f2f7b..78ddb395cca4 100644 --- a/platform/platform-resources/src/META-INF/LangExtensionPoints.xml +++ b/platform/platform-resources/src/META-INF/LangExtensionPoints.xml @@ -912,6 +912,10 @@ + + + diff --git a/platform/util/resources/misc/registry.properties b/platform/util/resources/misc/registry.properties index 49f455a4cae9..765bbacbe46e 100644 --- a/platform/util/resources/misc/registry.properties +++ b/platform/util/resources/misc/registry.properties @@ -1229,3 +1229,6 @@ batch.inspections.process.by.default.jvm.languages.description=When building gra junit4.search.4.tests.in.classpath=true junit4.search.4.tests.in.classpath.description=Search for tests in the module output + +use.prebuilt.stubs=true +use.prebuilt.stubs.description=Use bundled prebuilt stubs if they are available for better indexing performance \ No newline at end of file diff --git a/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy index 050eb6b1bfd9..6cf035d83c81 100644 --- a/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy +++ b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy @@ -45,6 +45,11 @@ abstract class PyCharmPropertiesBase extends ProductProperties { include(name: "*.pdf") } } + context.ant.copy(todir: "$targetDirectory/indexes") { + fileset(dir: "$context.paths.projectHome/indexes") { + include(name: "sdk-stubs*") + } + } } @Override diff --git a/python/psi-api/src/com/jetbrains/python/psi/LanguageLevel.java b/python/psi-api/src/com/jetbrains/python/psi/LanguageLevel.java index 71bf2cd784ae..741cb0b889a7 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/LanguageLevel.java +++ b/python/psi-api/src/com/jetbrains/python/psi/LanguageLevel.java @@ -23,6 +23,8 @@ import com.intellij.util.ArrayUtil; import org.jetbrains.annotations.NotNull; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @author yole @@ -40,7 +42,10 @@ public enum LanguageLevel { PYTHON35(35, true, false, true, true), PYTHON36(36, true, false, true, true); - public static List ALL_LEVELS = ImmutableList.copyOf(values()); + public static final List ALL_LEVELS = ImmutableList.copyOf(values()); + + public static final List SUPPORTED_LEVELS = ImmutableList.copyOf(Stream.of(values()).filter(v -> v.myVersion >= 26).collect( + Collectors.toList())); // Python versions 2.4 and 2.5 aren't supported anymore private static final LanguageLevel DEFAULT2 = PYTHON27; private static final LanguageLevel DEFAULT3 = PYTHON36; diff --git a/python/python-community-configure/resources/META-INF/python-community-configure-ide.xml b/python/python-community-configure/resources/META-INF/python-community-configure-ide.xml index 975d12d49ce3..5e74e86a874e 100644 --- a/python/python-community-configure/resources/META-INF/python-community-configure-ide.xml +++ b/python/python-community-configure/resources/META-INF/python-community-configure-ide.xml @@ -16,5 +16,7 @@ id="com.jetbrains.python.configuration.PyDependenciesConfigurable" displayName="Project Dependencies" provider="com.jetbrains.python.configuration.PyDependenciesConfigurableProvider"/> + + diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java index ae61d2f84005..fdb835686262 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java @@ -21,10 +21,7 @@ import com.intellij.openapi.extensions.Extensions; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.Ref; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.psi.PsiComment; -import com.intellij.psi.PsiElement; -import com.intellij.psi.PsiReference; -import com.intellij.psi.StubBasedPsiElement; +import com.intellij.psi.*; import com.intellij.psi.search.LocalSearchScope; import com.intellij.psi.search.SearchScope; import com.intellij.psi.stubs.IStubElementType; @@ -38,6 +35,7 @@ import com.intellij.util.containers.JBIterable; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.PythonDialectsTokenSetProvider; import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache; import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; @@ -151,7 +149,14 @@ public class PyFunctionImpl extends PyBaseElementImpl implements @Override @Nullable public ASTNode getNameNode() { - return getNode().findChildByType(PyTokenTypes.IDENTIFIER); + ASTNode id = getNode().findChildByType(PyTokenTypes.IDENTIFIER); + if (id == null) { + ASTNode error = getNode().findChildByType(TokenType.ERROR_ELEMENT); + if (error != null) { + id = error.findChildByType(PythonDialectsTokenSetProvider.INSTANCE.getKeywordTokens()); + } + } + return id; } @Override diff --git a/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java b/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java new file mode 100644 index 000000000000..222270c4dfb7 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java @@ -0,0 +1,112 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.psi.impl.stubs; + +import com.google.common.hash.HashCode; +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.psi.stubs.*; +import com.intellij.util.indexing.FileContent; +import com.intellij.util.io.PersistentHashMap; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; + +/** + * @author traff + */ +public class PyPrebuiltStubsProvider implements PrebuiltStubsProvider { + private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider"); + public static final String PREBUILT_INDEXES_PATH_PROPERTY = "prebuilt_indexes_path"; + + public static final String SDK_STUBS_STORAGE_NAME = "sdk-stubs"; + + private final FileContentHashing myFileContentHashing = new FileContentHashing(); + + private PersistentHashMap myPrebuiltStubsStorage; + private SerializationManagerImpl mySerializationManager; + + public PyPrebuiltStubsProvider() { + init(); + } + + public synchronized void init() { + File indexesRoot = findPrebuiltIndexesRoot(); + try { + if (indexesRoot != null) { + myPrebuiltStubsStorage = + new PersistentHashMap(new File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".input"), + HashCodeDescriptor.Companion.getInstance(), + new StubTreeExternalizer()) { + @Override + protected boolean isReadOnly() { + return true; + } + }; + mySerializationManager = new SerializationManagerImpl(new File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".names")); + + LOG.info("Using prebuilt stubs from " + myPrebuiltStubsStorage.getBaseFile().getAbsolutePath()); + } + } + catch (Exception e) { + myPrebuiltStubsStorage = null; + LOG.warn("Prebuilt stubs can't be loaded at " + indexesRoot, e); + } + } + + + @Nullable + @Override + public synchronized Stub findStub(@NotNull FileContent fileContent) { + if (myPrebuiltStubsStorage != null) { + HashCode hashCode = myFileContentHashing.hashString(fileContent); + Stub stub = null; + try { + SerializedStubTree stubTree = myPrebuiltStubsStorage.get(hashCode); + if (stubTree != null) { + stub = stubTree.getStub(false, mySerializationManager); + } + } + catch (SerializerNotFoundException e) { + LOG.error("Can't deserialize stub tree", e); + } + catch (Exception e) { + LOG.error("Error reading prebuilt stubs from " + myPrebuiltStubsStorage.getBaseFile().getPath(), e); + myPrebuiltStubsStorage = null; + stub = null; + } + if (stub != null) { + return stub; + } + } + return null; + } + + @Nullable + private static File findPrebuiltIndexesRoot() { + String path = System.getProperty(PREBUILT_INDEXES_PATH_PROPERTY); + if (path != null && new File(path).exists()) { + return new File(path); + } + path = PathManager.getHomePath(); + File f = new File(path, "python/indexes"); // from sources + if (f.exists()) return f; + f = new File(path, "indexes"); // compiled binary + if (f.exists()) return f; + return null; + } +} diff --git a/python/testSrc/com/jetbrains/python/PyUnifiedStubsTest.kt b/python/testSrc/com/jetbrains/python/PyUnifiedStubsTest.kt new file mode 100644 index 000000000000..0cfe8455d890 --- /dev/null +++ b/python/testSrc/com/jetbrains/python/PyUnifiedStubsTest.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python + +import com.intellij.psi.PsiManager +import com.jetbrains.python.fixtures.PyTestCase +import com.jetbrains.python.psi.LanguageLevel +import com.jetbrains.python.psi.PyFile +import junit.framework.TestCase + +/** + * @author traff + */ + +class PyUnifiedStubsTest : PyTestCase() { + + override fun getTestDataPath(): String { + return PythonTestUtil.getTestDataPath() + } + + fun testExecPy3() { + for (level in LanguageLevel.SUPPORTED_LEVELS) { + runWithLanguageLevel(level) { + val vf = myFixture.copyFileToProject("psi/${getTestName(false)}.py") + val file = PsiManager.getInstance(myFixture.project).findFile(vf) as PyFile + + TestCase.assertEquals("Python $level", "exec", file.topLevelFunctions[0].name) + } + } + } +} \ No newline at end of file diff --git a/python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt b/python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt new file mode 100644 index 000000000000..6c25e9179ffe --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt @@ -0,0 +1,160 @@ +package com.jetbrains.python.tools + +import com.google.common.hash.HashCode +import com.intellij.idea.IdeaTestApplication +import com.intellij.openapi.application.PathManager +import com.intellij.openapi.application.ReadAction +import com.intellij.openapi.application.WriteAction +import com.intellij.openapi.project.ProjectManager +import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil +import com.intellij.openapi.roots.OrderRootType +import com.intellij.openapi.util.Disposer +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream +import com.intellij.openapi.vfs.VfsUtilCore +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.openapi.vfs.VirtualFileVisitor +import com.intellij.psi.impl.DebugUtil +import com.intellij.psi.stubs.* +import com.intellij.util.indexing.FileContentImpl +import com.intellij.util.io.PersistentHashMap +import com.intellij.util.ui.UIUtil +import com.jetbrains.python.PythonFileType +import com.jetbrains.python.psi.LanguageLevel +import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider.SDK_STUBS_STORAGE_NAME +import junit.framework.TestCase +import java.io.ByteArrayInputStream +import java.io.File +import java.util.* + +/** + * @author traff + */ + +val stubsFileName = SDK_STUBS_STORAGE_NAME + +fun main(args: Array) { + val app = IdeaTestApplication.getInstance() + + + try { + indexSdkAndStoreSerializedStubs("${PathManager.getHomePath()}/python/testData/empty", + File(System.getenv("PYCHARM_PERF_ENVS"), "envs/py36_64").absolutePath, + "${System.getProperty("user.dir")}/$stubsFileName") + } + catch (e: Exception) { + e.printStackTrace() + } + finally { + UIUtil.invokeAndWaitIfNeeded(Runnable { + WriteAction.run { + app.dispose() + } + }) + System.exit(0) //TODO: graceful shutdown + } +} + +fun indexSdkAndStoreSerializedStubs(projectPath: String, sdkPath: String, stubsFilePath: String) { + val pair = openProjectWithSdk(projectPath, sdkPath) + + val project = pair.first + val sdk = pair.second + + val roots: List = sdk!!.rootProvider.getFiles(OrderRootType.CLASSES).asList() + + val hashing = FileContentHashing() + + + val stubExternalizer = StubTreeExternalizer() + val storage = PersistentHashMap(File(stubsFilePath + ".input"), + HashCodeDescriptor.instance, stubExternalizer) + + val serializationManager = SerializationManagerImpl(File(stubsFilePath + ".names")) + + try { + val map = HashMap>() + + for (file in roots) { + VfsUtilCore.visitChildrenRecursively(file, object : VirtualFileVisitor() { + override fun visitFile(file: VirtualFile): Boolean { + if (file.isDirectory && file.name == "parts") { + return false + } + if (file.fileType == PythonFileType.INSTANCE) { + val fileContent = FileContentImpl(file, file.contentsToByteArray()) + val stub = buildStubForFile(fileContent, serializationManager) + val hashCode = hashing.hashString(fileContent) + + val stub2 = storage.get(hashCode) + + val bytes = BufferExposingByteArrayOutputStream() + serializationManager.serialize(stub, bytes) + + + val contentLength = + if (file.fileType.isBinary) { + -1 + } + else { + fileContent.psiFileForPsiDependentIndex.textLength + } + + val stubTree = SerializedStubTree(bytes.internalBuffer, bytes.size(), stub, file.length, contentLength) + val item = map.get(hashCode) + if (item == null) { + storage.put(hashCode, stubTree) + map.put(hashCode, Pair(fileContent.contentAsText.toString(), stubTree)) + } + else { + TestCase.assertEquals(item.first, fileContent.contentAsText.toString()) + TestCase.assertTrue(stubTree == item.second) + } + } + + + return true + } + }) + } + } + finally { + storage.close() + + LanguageLevel.FORCE_LANGUAGE_LEVEL = LanguageLevel.getDefault() + UIUtil.invokeAndWaitIfNeeded(Runnable { + ProjectManager.getInstance().closeProject(project!!) + WriteAction.run { + Disposer.dispose(project) + SdkConfigurationUtil.removeSdk(sdk) + } + }) + } +} + +private fun buildStubForFile(fileContent: FileContentImpl, + serializationManager: SerializationManagerImpl): Stub { + var prevLanguageLevelBytes: ByteArray? = null + var prevLanguageLevel: LanguageLevel? = null + var prevStub: Stub? = null + for (languageLevel in LanguageLevel.SUPPORTED_LEVELS) { + LanguageLevel.FORCE_LANGUAGE_LEVEL = languageLevel + + val stub = ReadAction.compute { StubTreeBuilder.buildStubTree(fileContent) } + val bytes = BufferExposingByteArrayOutputStream() + serializationManager.serialize(stub!!, bytes) + + if (prevLanguageLevelBytes != null) { + if (!Arrays.equals(bytes.toByteArray(), prevLanguageLevelBytes)) { + val stub2 = serializationManager.deserialize(ByteArrayInputStream(prevLanguageLevelBytes)) + val msg = "Stubs are different for ${fileContent.file.path} between Python versions $prevLanguageLevel and $languageLevel.\n" + TestCase.assertEquals(msg, DebugUtil.stubTreeToString(stub), DebugUtil.stubTreeToString(stub2)) + TestCase.fail(msg + "But DebugUtil.stubTreeToString values of stubs are unfortunately equal.") + } + } + prevLanguageLevelBytes = bytes.toByteArray() + prevLanguageLevel = languageLevel + prevStub = stub + } + + return prevStub!! +}