Move python tools code to community

2026-09-27 10:03:11 +07:00 · 2017-08-18 20:58:24 +02:00
parent 3d9d53f104
commit ac01e7301c
10 changed files with 317 additions and 1 deletions
@@ -232,6 +232,7 @@
      <module fileurl="file://$PROJECT_DIR$/python/pluginMinor/python-community-plugin-minor.iml" filepath="$PROJECT_DIR$/python/pluginMinor/python-community-plugin-minor.iml" group="python" />
      <module fileurl="file://$PROJECT_DIR$/python/python-community-plugin-resources.iml" filepath="$PROJECT_DIR$/python/python-community-plugin-resources.iml" group="python" />
      <module fileurl="file://$PROJECT_DIR$/python/testSrc/python-community-tests.iml" filepath="$PROJECT_DIR$/python/testSrc/python-community-tests.iml" group="python" />
+      <module fileurl="file://$PROJECT_DIR$/python/tools/python-community-tools.iml" filepath="$PROJECT_DIR$/python/tools/python-community-tools.iml" group="python" />
      <module fileurl="file://$PROJECT_DIR$/python/educational-python/build/python-edu-build.iml" filepath="$PROJECT_DIR$/python/educational-python/build/python-edu-build.iml" group="python/educational" />
      <module fileurl="file://$PROJECT_DIR$/python/python-helpers.iml" filepath="$PROJECT_DIR$/python/python-helpers.iml" group="python" />
      <module fileurl="file://$PROJECT_DIR$/python/openapi/python-openapi.iml" filepath="$PROJECT_DIR$/python/openapi/python-openapi.iml" group="python" />
@@ -24,6 +24,6 @@
    <orderEntry type="library" exported="" scope="TEST" name="hamcrest" level="project" />
    <orderEntry type="module" module-name="xml-openapi" />
    <orderEntry type="module" module-name="lang-tests" scope="TEST" />
-    <orderEntry type="module" module-name="tools" scope="TEST" />
+    <orderEntry type="module" module-name="python-community-tools" scope="TEST" />
  </component>
 </module>
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
+    <orderEntry type="module" module-name="testFramework" />
+    <orderEntry type="module" module-name="python-psi-api" />
+    <orderEntry type="module" module-name="python-community" />
+    <orderEntry type="module" module-name="python-ide" exported="" scope="RUNTIME" />
+    <orderEntry type="module" module-name="python" exported="" />
+    <orderEntry type="library" name="Guava" level="project" />
+    <orderEntry type="module" module-name="python-helpers" />
+  </component>
+</module>
@@ -0,0 +1,16 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+
+plugins {
+  id "com.jetbrains.python.envs" version "0.0.17"
+}
+
+
+envs {
+  bootstrapDirectory = new File(System.getenv().getOrDefault("PYCHARM_PYTHONS", new File(buildDir, 'pythons').path))
+  envsDirectory = new File(System.getenv().getOrDefault("PYCHARM_PYTHON_VIRTUAL_ENVS", new File(buildDir, 'envs').path))
+  minicondaVersion = 'latest'
+  _64Bits = true
+
+  python "python27", "2.7.13", "64", []
+  python "python36", "3.6.2", "64", []
+}
@@ -0,0 +1,57 @@
+package com.jetbrains.python.tools
+
+import com.intellij.idea.IdeaTestApplication
+import com.intellij.openapi.vfs.VfsUtil
+import com.intellij.util.io.ZipUtil
+import com.jetbrains.python.sdk.PythonSdkType
+import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher
+import com.jetbrains.python.sdk.skeletons.SkeletonVersionChecker
+import com.jetbrains.python.tools.sdkTools.PySdkTools
+import com.jetbrains.python.tools.sdkTools.SdkCreationType
+import java.io.File
+import java.io.FileOutputStream
+import java.util.zip.ZipOutputStream
+
+/**
+ * @author traff
+ */
+
+val PYCHARM_PYTHONS = "PYCHARM_PYTHONS"
+
+fun main(args: Array<String>) {
+  val app = IdeaTestApplication.getInstance()
+  try {
+
+    val root = System.getenv(PYCHARM_PYTHONS)
+
+    for (python in File(root).listFiles()) {
+
+      println("Running on $python")
+
+      val sdk = PySdkTools.createTempSdk(VfsUtil.findFileByIoFile(File(PythonSdkType.getPythonExecutable(python.absolutePath)!!), true)!!,
+                                         SdkCreationType.SDK_PACKAGES_AND_SKELETONS, null)
+
+      val skeletonsDir = File(System.getProperty("user.dir"), "skeletons-${sdk.versionString!!.replace(" ", "_")}_" + +Math.abs(sdk.homePath!!.hashCode()))
+
+      println("Generating skeletons in ${skeletonsDir.absolutePath}")
+
+      val refresher = PySkeletonRefresher(null, null, sdk, skeletonsDir.absolutePath, null, null)
+
+
+      refresher.regenerateSkeletons(SkeletonVersionChecker(0))
+
+
+      val dirPacked = File(skeletonsDir.parent, refresher.pregeneratedSkeletonsName)
+      val zip = ZipOutputStream(FileOutputStream(dirPacked))
+      ZipUtil.addDirToZipRecursively(zip, dirPacked, skeletonsDir, "", null, null)
+      zip.close()
+    }
+  }
+  catch (e: Exception) {
+    e.printStackTrace()
+    System.exit(1)
+  }
+  finally {
+    System.exit(0) //TODO: graceful exit?
+  }
+}
@@ -0,0 +1,54 @@
+package com.jetbrains.python.tools
+
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.application.impl.ApplicationInfoImpl
+import com.intellij.openapi.module.Module
+import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ProjectManager
+import com.intellij.openapi.projectRoots.Sdk
+import com.intellij.openapi.roots.ModuleRootManager
+import com.intellij.openapi.roots.ModuleRootModificationUtil
+import com.intellij.openapi.vfs.VfsUtil
+import com.intellij.util.ui.UIUtil
+import com.jetbrains.python.sdk.PythonSdkType
+import com.jetbrains.python.sdk.PythonSdkUpdater
+import com.jetbrains.python.tools.sdkTools.PySdkTools
+import com.jetbrains.python.tools.sdkTools.SdkCreationType
+import java.io.File
+
+internal val TestPath = System.getenv("PYCHARM_PERF_ENVS")
+
+@JvmOverloads
+fun createSdkForPerformance(module: Module,
+                            sdkCreationType: SdkCreationType,
+                            sdkHome: String = File(TestPath, "envs/py36_64").absolutePath): Sdk {
+  ApplicationInfoImpl.setInStressTest(true) // To disable slow debugging
+  val executable = File(PythonSdkType.getPythonExecutable(sdkHome) ?: throw AssertionError("No python on $sdkHome"))
+  println("Running on $sdkHome")
+  return PySdkTools.createTempSdk(VfsUtil.findFileByIoFile(executable, true)!!, sdkCreationType, module)
+}
+
+
+fun openProjectWithSdk(projectPath: String, sdkHome: String): Pair<Project?, Sdk?> {
+  val project: Project? = ProjectManager.getInstance().loadAndOpenProject(projectPath)
+
+  val module = ModuleManager.getInstance(project!!).modules[0]
+
+  val sdk = createSdkForPerformance(module, SdkCreationType.SDK_PACKAGES_AND_SKELETONS, sdkHome)
+
+  UIUtil.invokeAndWaitIfNeeded(Runnable {
+    ApplicationManager.getApplication().runWriteAction({
+                                                         PythonSdkUpdater.update(sdk, null, project, null)
+                                                       })
+  })
+
+  if (module != null) {
+    ModuleRootModificationUtil.setModuleSdk(module, sdk)
+  }
+
+  assert(ModuleRootManager.getInstance(module).orderEntries().classesRoots.size > 5)
+  assert(ModuleManager.getInstance(project).modules.size == 1)
+
+  return Pair(project, sdk)
+}
@@ -0,0 +1,6 @@
+/**
+ * Engine to create SDK in tests.
+ * See {@link com.jetbrains.python.sdkTools.PyTestSdkTools}
+ * @author Ilya.Kazakevich
+ */
+package com.jetbrains.python.sdkTools;
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.python.tools.sdkTools;
+
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.application.PathManager;
+import com.intellij.openapi.module.Module;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.projectRoots.Sdk;
+import com.intellij.openapi.projectRoots.SdkModificator;
+import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil;
+import com.intellij.openapi.roots.ModuleRootModificationUtil;
+import com.intellij.openapi.roots.OrderRootType;
+import com.intellij.openapi.roots.ProjectRootManager;
+import com.intellij.openapi.util.Ref;
+import com.intellij.openapi.vfs.LocalFileSystem;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.jetbrains.python.sdk.InvalidSdkException;
+import com.jetbrains.python.sdk.PythonSdkAdditionalData;
+import com.jetbrains.python.sdk.PythonSdkType;
+import com.jetbrains.python.sdk.flavors.PythonSdkFlavor;
+import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
+
+import java.io.IOException;
+
+/**
+ * Engine to create SDK for tests.
+ * See {@link #createTempSdk(com.intellij.openapi.vfs.VirtualFile, SdkCreationType, com.intellij.openapi.module.Module)}
+ *
+ * @author Ilya.Kazakevich
+ */
+public final class PySdkTools {
+
+  private static final Sdk[] NO_SDK = new Sdk[0];
+
+  private PySdkTools() {
+
+  }
+
+  /**
+   * Creates SDK by its path and associates it with module (if module provided)
+   *
+   * @param sdkHome         path to sdk
+   * @param sdkCreationType SDK creation strategy (see {@link SdkCreationType} doc)
+   * @return sdk
+   */
+  @NotNull
+  public static Sdk createTempSdk(@NotNull final VirtualFile sdkHome,
+                                  @NotNull final SdkCreationType sdkCreationType,
+                                  @Nullable final Module module
+  )
+    throws InvalidSdkException {
+    final Ref<Sdk> ref = Ref.create();
+    ApplicationManager.getApplication().invokeAndWait(() -> {
+      final Sdk sdk = SdkConfigurationUtil.setupSdk(NO_SDK, sdkHome, PythonSdkType.getInstance(), true, null, null);
+      Assert.assertNotNull("Failed to create SDK on " + sdkHome, sdk);
+      ref.set(sdk);
+    });
+    final Sdk sdk = ref.get();
+    if (sdkCreationType != SdkCreationType.EMPTY_SDK) {
+      generateTempSkeletonsOrPackages(sdk, sdkCreationType == SdkCreationType.SDK_PACKAGES_AND_SKELETONS, module);
+    }
+    ApplicationManager.getApplication().invokeAndWait(() -> SdkConfigurationUtil.addSdk(sdk));
+    return sdk;
+  }
+
+
+  /**
+   * Adds installed eggs to SDK, generates skeletons (optionally) and associates it with module.
+   *
+   * @param sdk          sdk to process
+   * @param addSkeletons add skeletons or only packages
+   * @param module       module to associate with (if provided)
+   * @throws InvalidSdkException bas sdk
+   * @throws IOException         failed to read eggs
+   */
+  public static void generateTempSkeletonsOrPackages(@NotNull final Sdk sdk,
+                                                     final boolean addSkeletons,
+                                                     @Nullable final Module module)
+    throws InvalidSdkException {
+    Project project = null;
+
+    if (module != null) {
+      project = module.getProject();
+      final Project finalProject = project;
+      // Associate with module
+      ModuleRootModificationUtil.setModuleSdk(module, sdk);
+
+      ApplicationManager.getApplication().invokeAndWait(() -> ApplicationManager.getApplication()
+        .runWriteAction(() -> ProjectRootManager.getInstance(finalProject).setProjectSdk(sdk)));
+    }
+
+
+    final SdkModificator modificator = sdk.getSdkModificator();
+
+    modificator.setSdkAdditionalData(new PythonSdkAdditionalData(PythonSdkFlavor.getFlavor(sdk)));
+
+    for (final String path : PythonSdkType.getSysPathsFromScript(sdk.getHomePath())) {
+      addTestSdkRoot(modificator, path);
+    }
+    if (!addSkeletons) {
+      ApplicationManager.getApplication().invokeAndWait(modificator::commitChanges);
+      return;
+    }
+
+    final String skeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), sdk.getHomePath());
+    addTestSdkRoot(modificator, skeletonsPath);
+
+    ApplicationManager.getApplication().invokeAndWait(modificator::commitChanges);
+
+    PySkeletonRefresher
+      .refreshSkeletonsOfSdk(project, null, skeletonsPath, sdk);
+  }
+
+  public static void addTestSdkRoot(@NotNull SdkModificator sdkModificator, @NotNull String path) {
+    final VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByPath(path);
+    if (file != null) {
+      sdkModificator.addRoot(PythonSdkType.getSdkRootVirtualFile(file), OrderRootType.CLASSES);
+    }
+  }
+}
@@ -0,0 +1,20 @@
+package com.jetbrains.python.tools.sdkTools;
+
+/**
+ * SDK creation type
+ * @author Ilya.Kazakevich
+ */
+public enum SdkCreationType {
+  /**
+   * SDK only (no packages nor skeletons)
+   */
+  EMPTY_SDK,
+  /**
+   * SDK + installed packages from syspath
+   */
+  SDK_PACKAGES_ONLY,
+  /**
+   * SDK + installed packages from syspath + skeletons
+   */
+  SDK_PACKAGES_AND_SKELETONS
+}
@@ -0,0 +1,6 @@
+/**
+ * Engine to create SDK in tests.
+ * See {@link com.jetbrains.python.tools.PyTestSdkTools}
+ * @author Ilya.Kazakevich
+ */
+package com.jetbrains.python.tools.sdkTools;