diff --git a/.idea/modules.xml b/.idea/modules.xml index babe46bc8362..75f20ff89883 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -232,6 +232,7 @@ + diff --git a/python/testSrc/python-community-tests.iml b/python/testSrc/python-community-tests.iml index e842dc9895da..24a998f78039 100644 --- a/python/testSrc/python-community-tests.iml +++ b/python/testSrc/python-community-tests.iml @@ -24,6 +24,6 @@ - + \ No newline at end of file diff --git a/python/tools/python-community-tools.iml b/python/tools/python-community-tools.iml new file mode 100644 index 000000000000..bc4f05f78905 --- /dev/null +++ b/python/tools/python-community-tools.iml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/python/tools/setup-envs/build.gradle b/python/tools/setup-envs/build.gradle new file mode 100644 index 000000000000..30ff2a60c4f8 --- /dev/null +++ b/python/tools/setup-envs/build.gradle @@ -0,0 +1,16 @@ +import org.apache.tools.ant.taskdefs.condition.Os + +plugins { + id "com.jetbrains.python.envs" version "0.0.17" +} + + +envs { + bootstrapDirectory = new File(System.getenv().getOrDefault("PYCHARM_PYTHONS", new File(buildDir, 'pythons').path)) + envsDirectory = new File(System.getenv().getOrDefault("PYCHARM_PYTHON_VIRTUAL_ENVS", new File(buildDir, 'envs').path)) + minicondaVersion = 'latest' + _64Bits = true + + python "python27", "2.7.13", "64", [] + python "python36", "3.6.2", "64", [] +} diff --git a/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt b/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt new file mode 100644 index 000000000000..e410b711f659 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/BuildZippedSkeletons.kt @@ -0,0 +1,57 @@ +package com.jetbrains.python.tools + +import com.intellij.idea.IdeaTestApplication +import com.intellij.openapi.vfs.VfsUtil +import com.intellij.util.io.ZipUtil +import com.jetbrains.python.sdk.PythonSdkType +import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher +import com.jetbrains.python.sdk.skeletons.SkeletonVersionChecker +import com.jetbrains.python.tools.sdkTools.PySdkTools +import com.jetbrains.python.tools.sdkTools.SdkCreationType +import java.io.File +import java.io.FileOutputStream +import java.util.zip.ZipOutputStream + +/** + * @author traff + */ + +val PYCHARM_PYTHONS = "PYCHARM_PYTHONS" + +fun main(args: Array) { + val app = IdeaTestApplication.getInstance() + try { + + val root = System.getenv(PYCHARM_PYTHONS) + + for (python in File(root).listFiles()) { + + println("Running on $python") + + val sdk = PySdkTools.createTempSdk(VfsUtil.findFileByIoFile(File(PythonSdkType.getPythonExecutable(python.absolutePath)!!), true)!!, + SdkCreationType.SDK_PACKAGES_AND_SKELETONS, null) + + val skeletonsDir = File(System.getProperty("user.dir"), "skeletons-${sdk.versionString!!.replace(" ", "_")}_" + +Math.abs(sdk.homePath!!.hashCode())) + + println("Generating skeletons in ${skeletonsDir.absolutePath}") + + val refresher = PySkeletonRefresher(null, null, sdk, skeletonsDir.absolutePath, null, null) + + + refresher.regenerateSkeletons(SkeletonVersionChecker(0)) + + + val dirPacked = File(skeletonsDir.parent, refresher.pregeneratedSkeletonsName) + val zip = ZipOutputStream(FileOutputStream(dirPacked)) + ZipUtil.addDirToZipRecursively(zip, dirPacked, skeletonsDir, "", null, null) + zip.close() + } + } + catch (e: Exception) { + e.printStackTrace() + System.exit(1) + } + finally { + System.exit(0) //TODO: graceful exit? + } +} \ No newline at end of file diff --git a/python/tools/src/com/jetbrains/python/tools/Utils.kt b/python/tools/src/com/jetbrains/python/tools/Utils.kt new file mode 100644 index 000000000000..1a1e06f1d423 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/Utils.kt @@ -0,0 +1,54 @@ +package com.jetbrains.python.tools + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.impl.ApplicationInfoImpl +import com.intellij.openapi.module.Module +import com.intellij.openapi.module.ModuleManager +import com.intellij.openapi.project.Project +import com.intellij.openapi.project.ProjectManager +import com.intellij.openapi.projectRoots.Sdk +import com.intellij.openapi.roots.ModuleRootManager +import com.intellij.openapi.roots.ModuleRootModificationUtil +import com.intellij.openapi.vfs.VfsUtil +import com.intellij.util.ui.UIUtil +import com.jetbrains.python.sdk.PythonSdkType +import com.jetbrains.python.sdk.PythonSdkUpdater +import com.jetbrains.python.tools.sdkTools.PySdkTools +import com.jetbrains.python.tools.sdkTools.SdkCreationType +import java.io.File + +internal val TestPath = System.getenv("PYCHARM_PERF_ENVS") + +@JvmOverloads +fun createSdkForPerformance(module: Module, + sdkCreationType: SdkCreationType, + sdkHome: String = File(TestPath, "envs/py36_64").absolutePath): Sdk { + ApplicationInfoImpl.setInStressTest(true) // To disable slow debugging + val executable = File(PythonSdkType.getPythonExecutable(sdkHome) ?: throw AssertionError("No python on $sdkHome")) + println("Running on $sdkHome") + return PySdkTools.createTempSdk(VfsUtil.findFileByIoFile(executable, true)!!, sdkCreationType, module) +} + + +fun openProjectWithSdk(projectPath: String, sdkHome: String): Pair { + val project: Project? = ProjectManager.getInstance().loadAndOpenProject(projectPath) + + val module = ModuleManager.getInstance(project!!).modules[0] + + val sdk = createSdkForPerformance(module, SdkCreationType.SDK_PACKAGES_AND_SKELETONS, sdkHome) + + UIUtil.invokeAndWaitIfNeeded(Runnable { + ApplicationManager.getApplication().runWriteAction({ + PythonSdkUpdater.update(sdk, null, project, null) + }) + }) + + if (module != null) { + ModuleRootModificationUtil.setModuleSdk(module, sdk) + } + + assert(ModuleRootManager.getInstance(module).orderEntries().classesRoots.size > 5) + assert(ModuleManager.getInstance(project).modules.size == 1) + + return Pair(project, sdk) +} \ No newline at end of file diff --git a/python/tools/src/com/jetbrains/python/tools/package-info.java b/python/tools/src/com/jetbrains/python/tools/package-info.java new file mode 100644 index 000000000000..8b8fb74b9167 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/package-info.java @@ -0,0 +1,6 @@ +/** + * Engine to create SDK in tests. + * See {@link com.jetbrains.python.sdkTools.PyTestSdkTools} + * @author Ilya.Kazakevich + */ +package com.jetbrains.python.sdkTools; \ No newline at end of file diff --git a/python/tools/src/com/jetbrains/python/tools/sdkTools/PySdkTools.java b/python/tools/src/com/jetbrains/python/tools/sdkTools/PySdkTools.java new file mode 100644 index 000000000000..923cda15216b --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/sdkTools/PySdkTools.java @@ -0,0 +1,137 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.tools.sdkTools; + +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.module.Module; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.projectRoots.Sdk; +import com.intellij.openapi.projectRoots.SdkModificator; +import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil; +import com.intellij.openapi.roots.ModuleRootModificationUtil; +import com.intellij.openapi.roots.OrderRootType; +import com.intellij.openapi.roots.ProjectRootManager; +import com.intellij.openapi.util.Ref; +import com.intellij.openapi.vfs.LocalFileSystem; +import com.intellij.openapi.vfs.VirtualFile; +import com.jetbrains.python.sdk.InvalidSdkException; +import com.jetbrains.python.sdk.PythonSdkAdditionalData; +import com.jetbrains.python.sdk.PythonSdkType; +import com.jetbrains.python.sdk.flavors.PythonSdkFlavor; +import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; + +import java.io.IOException; + +/** + * Engine to create SDK for tests. + * See {@link #createTempSdk(com.intellij.openapi.vfs.VirtualFile, SdkCreationType, com.intellij.openapi.module.Module)} + * + * @author Ilya.Kazakevich + */ +public final class PySdkTools { + + private static final Sdk[] NO_SDK = new Sdk[0]; + + private PySdkTools() { + + } + + /** + * Creates SDK by its path and associates it with module (if module provided) + * + * @param sdkHome path to sdk + * @param sdkCreationType SDK creation strategy (see {@link SdkCreationType} doc) + * @return sdk + */ + @NotNull + public static Sdk createTempSdk(@NotNull final VirtualFile sdkHome, + @NotNull final SdkCreationType sdkCreationType, + @Nullable final Module module + ) + throws InvalidSdkException { + final Ref ref = Ref.create(); + ApplicationManager.getApplication().invokeAndWait(() -> { + final Sdk sdk = SdkConfigurationUtil.setupSdk(NO_SDK, sdkHome, PythonSdkType.getInstance(), true, null, null); + Assert.assertNotNull("Failed to create SDK on " + sdkHome, sdk); + ref.set(sdk); + }); + final Sdk sdk = ref.get(); + if (sdkCreationType != SdkCreationType.EMPTY_SDK) { + generateTempSkeletonsOrPackages(sdk, sdkCreationType == SdkCreationType.SDK_PACKAGES_AND_SKELETONS, module); + } + ApplicationManager.getApplication().invokeAndWait(() -> SdkConfigurationUtil.addSdk(sdk)); + return sdk; + } + + + /** + * Adds installed eggs to SDK, generates skeletons (optionally) and associates it with module. + * + * @param sdk sdk to process + * @param addSkeletons add skeletons or only packages + * @param module module to associate with (if provided) + * @throws InvalidSdkException bas sdk + * @throws IOException failed to read eggs + */ + public static void generateTempSkeletonsOrPackages(@NotNull final Sdk sdk, + final boolean addSkeletons, + @Nullable final Module module) + throws InvalidSdkException { + Project project = null; + + if (module != null) { + project = module.getProject(); + final Project finalProject = project; + // Associate with module + ModuleRootModificationUtil.setModuleSdk(module, sdk); + + ApplicationManager.getApplication().invokeAndWait(() -> ApplicationManager.getApplication() + .runWriteAction(() -> ProjectRootManager.getInstance(finalProject).setProjectSdk(sdk))); + } + + + final SdkModificator modificator = sdk.getSdkModificator(); + + modificator.setSdkAdditionalData(new PythonSdkAdditionalData(PythonSdkFlavor.getFlavor(sdk))); + + for (final String path : PythonSdkType.getSysPathsFromScript(sdk.getHomePath())) { + addTestSdkRoot(modificator, path); + } + if (!addSkeletons) { + ApplicationManager.getApplication().invokeAndWait(modificator::commitChanges); + return; + } + + final String skeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), sdk.getHomePath()); + addTestSdkRoot(modificator, skeletonsPath); + + ApplicationManager.getApplication().invokeAndWait(modificator::commitChanges); + + PySkeletonRefresher + .refreshSkeletonsOfSdk(project, null, skeletonsPath, sdk); + } + + public static void addTestSdkRoot(@NotNull SdkModificator sdkModificator, @NotNull String path) { + final VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByPath(path); + if (file != null) { + sdkModificator.addRoot(PythonSdkType.getSdkRootVirtualFile(file), OrderRootType.CLASSES); + } + } +} diff --git a/python/tools/src/com/jetbrains/python/tools/sdkTools/SdkCreationType.java b/python/tools/src/com/jetbrains/python/tools/sdkTools/SdkCreationType.java new file mode 100644 index 000000000000..430bb1b109f2 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/sdkTools/SdkCreationType.java @@ -0,0 +1,20 @@ +package com.jetbrains.python.tools.sdkTools; + +/** + * SDK creation type + * @author Ilya.Kazakevich + */ +public enum SdkCreationType { + /** + * SDK only (no packages nor skeletons) + */ + EMPTY_SDK, + /** + * SDK + installed packages from syspath + */ + SDK_PACKAGES_ONLY, + /** + * SDK + installed packages from syspath + skeletons + */ + SDK_PACKAGES_AND_SKELETONS +} diff --git a/python/tools/src/com/jetbrains/python/tools/sdkTools/package-info.java b/python/tools/src/com/jetbrains/python/tools/sdkTools/package-info.java new file mode 100644 index 000000000000..07c9c3b28b90 --- /dev/null +++ b/python/tools/src/com/jetbrains/python/tools/sdkTools/package-info.java @@ -0,0 +1,6 @@ +/** + * Engine to create SDK in tests. + * See {@link com.jetbrains.python.tools.PyTestSdkTools} + * @author Ilya.Kazakevich + */ +package com.jetbrains.python.tools.sdkTools; \ No newline at end of file