MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-04-19 21:11:28 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-43773 More robust synchronization of Python SDK background updates

Browse Source 
Previous synchronization of background updates in PythonSdkUpdater was flawed.
It didn't prevent us from launching several concurrent updates for the same
SDK, but only blocked subsequent refresh tasks on a BlockingSet, still showing
the corresponding progress indicators, piling up threads and exhausting system
resources.

The new implementation attempts to enforce the following policy: no two updates
for the same SDK can be run concurrently, and all subsequent updates for an SDK
already being refreshed are squashed and queued to be executed once it finished.

We also adjusted PythonSdkUpdater interface to better indicate synchronization
of individual methods. The old update() method, which performed a few of update
operations in a blocking fashion, is now deprecated and replaced with more
explicit scheduleUpdate(), always asynchronous, and internal
updateVersionAndPathsSynchronouslyAndScheduleRemaining() that replicates the old
behavior for the time being due to a number of existing usages, but is to be
revised. Synchronous version and paths refreshes are now performed under a modal
progress indicator.

The legacy updating mechanism can be restored as a fallback if
"python.use.new.sdk.updater" Registry flag is reset.

Additionally, internal PyUpdateProjectSdk action was added for diagnostic.

These changes are a result of a joint effort with Alexey Kniazev.

GitOrigin-RevId: 6260cda7a22c4f5932f7d78eb6660a50e2b972b6
This commit is contained in:
Mikhail Golubev
2020-06-26 13:54:00 +03:00
committed by intellij-monorepo-bot
parent 3f6bc5c1b9
commit 7b6cc1fe1f
7 changed files with 904 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/pluginJava/com/jetbrains/python/inspections/PythonPluginCommandLineInspectionProjectConfigurator.java
View File

@@ -2,9 +2,9 @@
package com.jetbrains.python.inspections;
import com.intellij.analysis.AnalysisScope;
import com.intellij.facet.FacetManager;
import com.intellij.ide.CommandLineInspectionProgressReporter;
import com.intellij.ide.CommandLineInspectionProjectConfigurator;
import com.intellij.facet.FacetManager;
import com.intellij.openapi.application.WriteAction;
import com.intellij.openapi.fileTypes.FileTypeRegistry;
import com.intellij.openapi.module.Module;
@@ -24,7 +24,6 @@ import org.jetbrains.annotations.NotNull;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
public class PythonPluginCommandLineInspectionProjectConfigurator implements CommandLineInspectionProjectConfigurator {
@@ -80,7 +79,7 @@ public class PythonPluginCommandLineInspectionProjectConfigurator implements Com
ProjectJdkTable.getInstance().addJdk(sdk);
});
PythonSdkUpdater.update(sdk, null, null, null);
PythonSdkUpdater.updateVersionAndPathsSynchronouslyAndScheduleRemaining(sdk, null, null);
}
else {
logger.reportMessage(1, "ERROR: Can't find Python interpreter");

2
python/pluginResources/messages/PyBundle.properties
View File

@@ -467,6 +467,8 @@ sdk.scanning.installed.packages=Scanning Installed Packages
sdk.gen.stubs.for.binary.modules=Generate stubs for binary module {0}
sdk.gen.failed.notification.title=Couldn't refresh skeletons for remote interpreter
sdk.gen.cannot.create.skeleton.dir=Can''t create skeleton dir {0}
sdk.updating.interpreter.paths=Updating interpreter paths...
sdk.updating.interpreter.version=Updating interpreter version...
# Active SDK configurable and related dialogs
active.sdk.dialog.show.all.item=Show All...

9
python/src/META-INF/python-core-common.xml
View File

@@ -368,6 +368,7 @@
<applicationService serviceInterface="com.jetbrains.python.codeInsight.imports.ImportChooser" serviceImplementation="com.jetbrains.python.codeInsight.imports.PyImportChooser"/>
<registryKey key="python.relative.import.depth" defaultValue="3" description="Specifies default acceptable number of dots in a relative import statement"/>
<registryKey key="python.use.new.sdk.updater" defaultValue="true" description="Use experimental PyUpdateSdkTask"/>
<!-- flake8 -->
<lang.inspectionSuppressor language="Python" implementationClass="com.jetbrains.python.inspections.flake8.Flake8InspectionSuppressor"/>
@@ -625,6 +626,14 @@
<add-to-group group-id="DebugMainMenu" relative-to-action="ForceRunToCursor" anchor="after"/>
<add-to-group group-id="EditorPopupMenuDebug" relative-to-action="ForceRunToCursor" anchor="after"/>
</action>
<!--suppress PluginXmlI18n -->
<group id="Internal.Python" internal="true" popup="true" text="Python">
<!--suppress PluginXmlI18n -->
<action id="PyUpdateProjectSdk" internal="true" class="com.jetbrains.python.sdk.PyUpdateProjectSdkAction"
text="Update Python SDK" description="Forcibly update all configured Python SDKs in the project"/>
<add-to-group group-id="Internal"/>
</group>
</actions>
<extensions defaultExtensionNs="com.intellij.spellchecker">

6
python/src/com/jetbrains/python/configuration/PythonSdkDetailsDialog.java
View File

@@ -393,8 +393,10 @@ public class PythonSdkDetailsDialog extends DialogWrapper {
/* PythonSdkUpdater.update invalidates the modificator so we need to create a new
one for further changes
*/
if (PythonSdkUpdater.update(currentSdk, myModificators.get(currentSdk), myProject, null)) {
myModifiedModificators.remove(myModificators.get(currentSdk));
SdkModificator modificator = myModificators.get(currentSdk);
assert modificator != null : "Modificator cannot be null here";
if (PythonSdkUpdater.updateVersionAndPathsSynchronouslyAndScheduleRemaining(currentSdk, modificator, myProject)){
myModifiedModificators.remove(modificator);
myModificators.put(currentSdk, currentSdk.getSdkModificator());
}
}

38
python/src/com/jetbrains/python/sdk/PyUpdateProjectSdkAction.java Normal file
View File

@@ -0,0 +1,38 @@
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.sdk;
import com.intellij.openapi.actionSystem.AnActionEvent;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.module.Module;
import com.intellij.openapi.module.ModuleManager;
import com.intellij.openapi.project.DumbAwareAction;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.projectRoots.Sdk;
import com.intellij.util.TimeoutUtil;
import org.jetbrains.annotations.NotNull;
public class PyUpdateProjectSdkAction extends DumbAwareAction {
private static final int N_REPEATS = 1;
private static final int TIMEOUT = 0; // ms
private static final Logger LOG = Logger.getInstance(PyUpdateProjectSdkAction.class);
@Override
public void actionPerformed(@NotNull AnActionEvent e) {
Project project = e.getProject();
if (project == null) return;
ApplicationManager.getApplication().executeOnPooledThread(() -> {
for (int i = 0; i < N_REPEATS; i++) {
for (Module module : ModuleManager.getInstance(project).getModules()) {
Sdk sdk = PythonSdkUtil.findPythonSdk(module);
if (sdk == null) {
LOG.info("Skipping module " + module + " as not having a Python SDK");
continue;
}
PythonSdkUpdater.scheduleUpdate(sdk, project);
TimeoutUtil.sleep(TIMEOUT);
}
}
});
}
}

455
python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java
View File

@@ -19,6 +19,7 @@ import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.Task;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.projectRoots.ProjectJdkTable;
import com.intellij.openapi.projectRoots.Sdk;
import com.intellij.openapi.projectRoots.SdkAdditionalData;
import com.intellij.openapi.projectRoots.SdkModificator;
@@ -26,13 +27,13 @@ import com.intellij.openapi.roots.OrderRootType;
import com.intellij.openapi.startup.StartupActivity;
import com.intellij.openapi.ui.Messages;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.StandardFileSystems;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.PathMappingSettings;
import com.intellij.util.Processor;
import com.intellij.util.concurrency.BlockingSet;
import com.jetbrains.python.PyBundle;
import com.jetbrains.python.codeInsight.typing.PyTypeShed;
import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil;
@@ -41,11 +42,13 @@ import com.jetbrains.python.psi.PyUtil;
import com.jetbrains.python.remote.PyRemoteSdkAdditionalDataBase;
import com.jetbrains.python.remote.UnsupportedPythonSdkTypeException;
import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.File;
import java.time.Instant;
import java.util.List;
import java.util.*;
import java.util.concurrent.Future;
@@ -60,60 +63,292 @@ public class PythonSdkUpdater implements StartupActivity.Background {
private static final Logger LOG = Logger.getInstance(PythonSdkUpdater.class);
private static final Object ourLock = new Object();
private static final Set<String> ourScheduledToRefresh = new HashSet<String>();
private static final BlockingSet<String> ourUnderRefresh = new BlockingSet<>();
private static final Set<String> ourUnderRefresh = new HashSet<>();
private static final Map<String, PyUpdateSdkRequestData> ourToBeRefreshed = new HashMap<>();
private static final NotificationGroup NOTIFICATION_GROUP = NotificationGroup.balloonGroup(
static final NotificationGroup NOTIFICATION_GROUP = NotificationGroup.balloonGroup(
"Python SDK Updater",
PyBundle.message("python.sdk.updater.notifications.group.title"));
/**
* Refreshes the SDKs of the modules for the open project after some delay.
* Schedules a background refresh of the SDKs of the modules for the open project.
*/
@Override
public void runActivity(@NotNull final Project project) {
public void runActivity(@NotNull Project project) {
if (!Registry.is("python.use.new.sdk.updater")) {
PythonSdkUpdaterOld.updateProjectSdksOnStartup(project);
return;
}
Application application = ApplicationManager.getApplication();
if (application.isUnitTestMode()) return;
if (project.isDisposed()) return;
ProgressManager.getInstance().run(new Task.Backgroundable(project, PyBundle.message("python.sdk.updating.python.paths"), false) {
@Override
public void run(@NotNull ProgressIndicator indicator) {
for (Sdk sdk : getPythonSdks(project)) {
update(sdk, null, project, null);
for (Sdk sdk : getPythonSdks(project)) {
scheduleUpdate(sdk, project);
}
}
private static class PyUpdateSdkRequestData {
final Instant myTimestamp;
final Throwable myTraceback;
private PyUpdateSdkRequestData() {
this(Instant.now(), new Throwable());
}
private PyUpdateSdkRequestData(@NotNull Instant timestamp, @NotNull Throwable traceback) {
myTimestamp = timestamp;
myTraceback = traceback;
}
@NotNull
private static PyUpdateSdkRequestData merge(@NotNull PyUpdateSdkRequestData oldRequest,
@NotNull PyUpdateSdkRequestData newRequest) {
return new PyUpdateSdkRequestData(oldRequest.myTimestamp, newRequest.myTraceback);
}
}
private static class PyUpdateSdkTask extends Task.Backgroundable {
private final @NotNull String mySdkKey;
private final @NotNull PyUpdateSdkRequestData myRequestData;
PyUpdateSdkTask(@NotNull Project project,
@NotNull String key,
@NotNull PyUpdateSdkRequestData requestData) {
super(project, PyBundle.message("sdk.gen.updating.interpreter"), false);
mySdkKey = key;
myRequestData = requestData;
}
@Override
public void run(@NotNull ProgressIndicator indicator) {
if (myProject.isDisposed()) {
return;
}
@Nullable Sdk sdk = PythonSdkUtil.findSdkByKey(mySdkKey);
if (sdk == null) {
LOG.warn("SDK for " + mySdkKey + " was removed from the SDK list");
return;
}
if (Trigger.LOG.isDebugEnabled()) {
Trigger.LOG.debug("Starting SDK refresh for '" + mySdkKey + "' triggered by " + Trigger.getCauseByTrace(myRequestData.myTraceback));
}
try {
activateVirtualEnv(sdk);
updateLocalSdkVersionAndPaths(sdk, null, myProject);
generateSkeletons(sdk, indicator);
refreshPackages(sdk, indicator);
}
catch (InvalidSdkException e) {
LOG.warn("Update for SDK " + sdk + " failed", e);
}
// restart code analysis
ApplicationManager.getApplication().invokeLater(() -> DaemonCodeAnalyzer.getInstance(myProject).restart(), myProject.getDisposed());
}
private void refreshPackages(@NotNull Sdk sdk, @NotNull ProgressIndicator indicator) {
try {
LOG.info("Performing background scan of packages for SDK " + getSdkPresentableName(sdk));
indicator.setIndeterminate(true);
indicator.setText(PyBundle.message("python.sdk.scanning.installed.packages"));
indicator.setText2("");
PyPackageManager.getInstance(sdk).refreshAndGetPackages(true);
}
catch (ExecutionException e) {
if (LOG.isDebugEnabled()) {
e.initCause(myRequestData.myTraceback);
LOG.debug(e);
}
else {
LOG.warn(e.getMessage());
}
}
});
}
private void generateSkeletons(@NotNull Sdk sdk, @NotNull ProgressIndicator indicator) {
final String skeletonsPath = PythonSdkUtil.getSkeletonsPath(sdk);
try {
final String sdkPresentableName = getSdkPresentableName(sdk);
LOG.info("Performing background update of skeletons for SDK " + sdkPresentableName);
indicator.setText(PyBundle.message("python.sdk.updating.skeletons"));
PySkeletonRefresher.refreshSkeletonsOfSdk(myProject, null, skeletonsPath, sdk);
updateRemoteSdkPaths(sdk, getProject());
}
catch (UnsupportedPythonSdkTypeException e) {
NOTIFICATION_GROUP
.createNotification(PyBundle.message("sdk.gen.failed.notification.title"), null,
PyBundle.message("remote.interpreter.support.is.not.available", sdk.getName()),
NotificationType.WARNING)
.notify(myProject);
}
catch (InvalidSdkException e) {
if (PythonSdkUtil.isRemote(mySdkKey)) {
PythonSdkType.notifyRemoteSdkSkeletonsFail(e, () -> {
Sdk revalidatedSdk = PythonSdkUtil.findSdkByKey(mySdkKey);
if (revalidatedSdk != null) {
update(revalidatedSdk, null, myProject, null);
}
});
}
else if (!PythonSdkUtil.isInvalid(sdk)) {
LOG.error(e);
}
}
}
@Override
public void onFinished() {
if (Trigger.LOG.isDebugEnabled()) {
Trigger.LOG.debug("Finishing SDK refresh for '" + mySdkKey + "' " +
"originally scheduled at " + myRequestData.myTimestamp + " by " +
Trigger.getCauseByTrace(myRequestData.myTraceback));
}
PyUpdateSdkRequestData requestData;
synchronized (ourLock) {
boolean existed = ourUnderRefresh.remove(mySdkKey);
LOG.assertTrue(existed, "Error in SDK refresh scheduling: refreshed SDK is not in the set.");
requestData = ourToBeRefreshed.remove(mySdkKey);
if (requestData != null) {
ourUnderRefresh.add(mySdkKey);
}
}
if (requestData != null) {
ProgressManager.getInstance().run(new PyUpdateSdkTask(myProject, mySdkKey, requestData));
}
}
}
/**
* Updates the paths of an SDK and regenerates its skeletons as a background task.
*
* May be invoked from any thread. May freeze the current thread while evaluating sys.path.
*
* For a local SDK it commits all the SDK paths and runs a background task for updating skeletons. For a remote SDK it runs a background
* task for updating skeletons that saves path mappings in the additional SDK data and then commits all the SDK paths.
*
* The commit of the changes in the SDK happens in the AWT thread while the current thread is waiting the result.
* @deprecated Use {@link #scheduleUpdate} or {@link #updateVersionAndPathsSynchronouslyAndScheduleRemaining}
*/
@ApiStatus.ScheduledForRemoval(inVersion = "2021.3")
@Deprecated
public static boolean update(@NotNull Sdk sdk,
@Nullable SdkModificator sdkModificator,
@Nullable Project project,
@Nullable Component ownerComponent) {
if (!Registry.is("python.use.new.sdk.updater")) {
return PythonSdkUpdaterOld.update(sdk, sdkModificator, project, ownerComponent);
}
return updateVersionAndPathsSynchronouslyAndScheduleRemaining(sdk, sdkModificator, project);
}
/**
* <i>Synchronously</i> update an interpreter version and paths in {@link ProjectJdkTable} and schedule a full-scale background refresh
* with {@link #scheduleUpdate(Sdk, Project)}.
* <p>
* For a local SDK, any version and paths changes are automatically committed. For a remote SDK, paths and path mappings are queried
* and saved in the background task after the skeleton generation finishes.
* <p>
* Since this method blocks for the first phase of an update, it's not allowed to call it on threads holding a read or write action.
* The only exception is made for EDT, in which case a modal progress indicator will be displayed during this first synchronous step.
* <p>
* This method emulates the legacy behavior of {@link #update(Sdk, SdkModificator, Project, Component)} and is likely to be removed
* or changed in future. Unless you're sure that a synchronous update is necessary you should rather use
* {@link #scheduleUpdate(Sdk, Project)} directly.
*
* @param sdkModificator if null then it tries to get an SDK modifier from the SDK table, falling back to the modifier of the SDK
* passed as an argument accessed from the AWT thread
* @return false if there was an immediate problem updating the SDK. Other problems are reported as log entries and balloons.
* @see #scheduleUpdate(Sdk, Project)
*/
public static boolean update(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable final Project project,
@Nullable final Component ownerComponent) {
final Application application = ApplicationManager.getApplication();
assert !application.isWriteAccessAllowed() : "sdk update should not be run in edt neither under write action";
final String key = PythonSdkType.getSdkKey(sdk);
synchronized (ourLock) {
ourScheduledToRefresh.add(key);
@ApiStatus.Internal
public static boolean updateVersionAndPathsSynchronouslyAndScheduleRemaining(@NotNull Sdk sdk,
@Nullable SdkModificator sdkModificator,
@Nullable Project project) {
if (!Registry.is("python.use.new.sdk.updater")) {
return PythonSdkUpdaterOld.update(sdk, sdkModificator, project, null);
}
SdkModificator effectiveModificator = sdkModificator != null ? sdkModificator : sdk.getSdkModificator();
Application application = ApplicationManager.getApplication();
try {
// This is not optimal but already happens in many contexts including possible external usages, e.g. during a new SDK generation.
if (application.isDispatchThread()) {
ProgressManager.getInstance().runProcessWithProgressSynchronously(() -> {
updateLocalSdkVersionAndPaths(sdk, effectiveModificator, project);
return null;
}, PyBundle.message("sdk.gen.updating.interpreter"), false, project);
}
else {
LOG.assertTrue(!application.isReadAccessAllowed(), "Synchronous SDK update should not be run under read action");
updateLocalSdkVersionAndPaths(sdk, effectiveModificator, project);
}
}
catch (InvalidSdkException e) {
LOG.warn("Error while evaluating path and version: ", e);
return false;
}
if (project == null) {
return true;
}
// Don't inline this variable, it needs to anchor the current stack.
PyUpdateSdkRequestData request = new PyUpdateSdkRequestData();
// When a new interpreter is still being generated, we need to wait until it finishes and SDK
// is properly written in ProjectJdkTable. Otherwise, a concurrent background update might fail.
boolean isSavedSdk = PythonSdkUtil.findSdkByKey(PythonSdkType.getSdkKey(sdk)) != null;
if (application.isWriteThread() && !isSavedSdk) {
application.invokeLaterOnWriteThread(() -> scheduleUpdate(sdk, project, request));
}
else {
scheduleUpdate(sdk, project, request);
}
return true;
}
/**
* Schedule an <i>asynchronous</i> background update of the given SDK.
* <p>
* This method may be invoked from any thread. Synchronization guarantees the following properties:
* <ul>
* <li>No two updates of the same SDK can be performed simultaneously.</li>
* <li>Subsequent requests to update an SDK already being refreshed will be queued and launched as soon as the ongoing update finishes.</li>
* <li>Multiple subsequent requests to update an SDK already being refreshed will be combined and result in a single update operation.</li>
* </ul>
*/
@ApiStatus.Experimental
public static void scheduleUpdate(@NotNull Sdk sdk, @NotNull Project project) {
if (!Registry.is("python.use.new.sdk.updater")) {
PythonSdkUpdaterOld.update(sdk, null, project, null);
return;
}
scheduleUpdate(sdk, project, new PyUpdateSdkRequestData());
}
private static void scheduleUpdate(@NotNull Sdk sdk, @NotNull Project project, @NotNull PyUpdateSdkRequestData requestData) {
if (ApplicationManager.getApplication().isUnitTestMode()) {
LOG.info("Skipping background update for '" + sdk + "' in unit test mode");
return;
}
final String key = PythonSdkType.getSdkKey(sdk);
synchronized (ourLock) {
if (ourUnderRefresh.contains(key)) {
if (Trigger.LOG.isDebugEnabled()) {
PyUpdateSdkRequestData previousRequest = ourToBeRefreshed.get(key);
if (previousRequest != null) {
String cause = Trigger.getCauseByTrace(previousRequest.myTraceback);
Trigger.LOG.debug("Discarding previous update for " + sdk + " triggered by " + cause);
}
}
ourToBeRefreshed.merge(key, requestData, PyUpdateSdkRequestData::merge);
return;
}
else {
ourUnderRefresh.add(key);
}
}
ProgressManager.getInstance().run(new PyUpdateSdkTask(project, key, requestData));
}
private static void activateVirtualEnv(@NotNull Sdk sdk) {
String sdkHome = sdk.getHomePath();
if (sdkHome != null && (PythonSdkUtil.isVirtualEnv(sdkHome) || PythonSdkUtil.isConda(sdk))) {
final Future<?> updateSdkFeature = application.executeOnPooledThread(() -> {
if (PythonSdkUtil.isVirtualEnv(sdkHome) || PythonSdkUtil.isConda(sdkHome)) {
final Future<?> updateSdkFeature = ApplicationManager.getApplication().executeOnPooledThread(() -> {
PySdkUtil.activateVirtualEnv(sdk); // pre-cache virtualenv activated environment
});
if (ApplicationManager.getApplication().isUnitTestMode()) {
@@ -126,132 +361,37 @@ public class PythonSdkUpdater implements StartupActivity.Background {
}
}
}
updateLocalSdkVersion(sdk, sdkModificator);
if (!updateLocalSdkPaths(sdk, sdkModificator, project)) {
return false;
}
if (project == null) {
return true;
}
if (application.isUnitTestMode()) {
// All actions we take after this line are dedicated to skeleton update process. Not all tests do need them. To find test API that
// updates skeleton, see PySkeletonRefresher
return true;
}
final Throwable methodCallStacktrace = new Throwable("SDK update trace");
application.invokeLater(() -> {
synchronized (ourLock) {
if (!ourScheduledToRefresh.contains(key)) {
if (Trigger.LOG.isDebugEnabled()) {
Trigger.LOG.debug("Dropping simultaneous SDK update triggered by " + Trigger.getCauseByTrace(methodCallStacktrace));
}
return;
}
ourScheduledToRefresh.remove(key);
}
if (project.isDisposed()) {
return;
}
if (PythonSdkUtil.findSdkByKey(key) == null) {
return;
}
ProgressManager.getInstance().run(new Task.Backgroundable(project, PyBundle.message("sdk.gen.updating.interpreter"), false) {
@Override
public void run(@NotNull ProgressIndicator indicator) {
final Project project1 = getProject();
final Sdk sdkInsideTask = PythonSdkUtil.findSdkByKey(key);
if (sdkInsideTask != null) {
ourUnderRefresh.put(key);
try {
final String skeletonsPath = PythonSdkUtil.getSkeletonsPath(sdk);
try {
if (PythonSdkUtil.isRemote(sdkInsideTask) && project1 == null && ownerComponent == null) {
LOG.error("For refreshing skeletons of remote SDK, either project or owner component must be specified");
}
final String sdkPresentableName = getSdkPresentableName(sdk);
LOG.info("Performing background update of skeletons for SDK " + sdkPresentableName);
indicator.setText(PyBundle.message("python.sdk.updating.skeletons"));
try {
if (Trigger.LOG.isDebugEnabled()) {
Trigger.LOG.debug("Performing skeletons update triggered by " + Trigger.getCauseByTrace(methodCallStacktrace));
}
PySkeletonRefresher.refreshSkeletonsOfSdk(project1, ownerComponent, skeletonsPath, sdkInsideTask);
updateRemoteSdkPaths(sdkInsideTask, getProject());
indicator.setIndeterminate(true);
indicator.setText(PyBundle.message("python.sdk.scanning.installed.packages"));
indicator.setText2("");
LOG.info("Performing background scan of packages for SDK " + sdkPresentableName);
PyPackageManager.getInstance(sdkInsideTask).refreshAndGetPackages(true);
}
catch (ExecutionException e) {
if (LOG.isDebugEnabled()) {
e.initCause(methodCallStacktrace);
LOG.debug(e);
}
else {
LOG.warn(e.getMessage());
}
}
if (project1 != null) {
application.invokeLater(() -> DaemonCodeAnalyzer.getInstance(project1).restart(), project1.getDisposed());
}
}
catch (UnsupportedPythonSdkTypeException e) {
NOTIFICATION_GROUP
.createNotification(PyBundle.message("sdk.gen.failed.notification.title"), null,
PyBundle.message("remote.interpreter.support.is.not.available", sdk.getName()),
NotificationType.WARNING)
.notify(project1);
}
catch (InvalidSdkException e) {
if (PythonSdkUtil.isRemote(sdkInsideTask)) {
PythonSdkType.notifyRemoteSdkSkeletonsFail(e, () -> {
final Sdk sdkInsideNotify = PythonSdkUtil.findSdkByKey(key);
if (sdkInsideNotify != null) {
update(sdkInsideNotify, null, project1, ownerComponent);
}
});
}
else if (!PythonSdkUtil.isInvalid(sdkInsideTask)) {
LOG.error(e);
}
}
}
finally {
try {
ourUnderRefresh.remove(key);
}
catch (IllegalStateException e) {
LOG.error(e);
}
}
}
}
});
}, ModalityState.NON_MODAL);
return true;
}
/**
* Updates the paths of an SDK and regenerates its skeletons as a background task. Shows an error message if the update fails.
* Updates the SDK as {@link #updateVersionAndPathsSynchronouslyAndScheduleRemaining(Sdk, SdkModificator, Project)} describes, but
* shows an error message if the first synchronous part of the update fails.
*
* @see #update(Sdk, SdkModificator, Project, Component)
* @see #updateVersionAndPathsSynchronouslyAndScheduleRemaining(Sdk, SdkModificator, Project)
*/
public static void updateOrShowError(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable Project project,
@ApiStatus.Internal
public static void updateOrShowError(@NotNull Sdk sdk,
@Nullable SdkModificator sdkModificator,
@Nullable Project project,
@Nullable Component ownerComponent) {
final boolean success = update(sdk, sdkModificator, project, ownerComponent);
if (!success) {
if (!Registry.is("python.use.new.sdk.updater")) {
PythonSdkUpdaterOld.updateOrShowError(sdk, sdkModificator, project, ownerComponent);
return;
}
boolean versionAndPathsUpdated = updateVersionAndPathsSynchronouslyAndScheduleRemaining(sdk, sdkModificator, project);
if (!versionAndPathsUpdated) {
Messages.showErrorDialog(project,
PyBundle.message("python.sdk.cannot.setup.sdk", getSdkPresentableName(sdk)),
PyBundle.message("python.sdk.invalid.python.sdk"));
}
}
private static void updateLocalSdkVersionAndPaths(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable Project project)
throws InvalidSdkException {
updateLocalSdkVersion(sdk, sdkModificator);
updateLocalSdkPaths(sdk, sdkModificator, project);
}
/**
* Changes the version string of an SDK if it's out of date.
* <p>
@@ -260,6 +400,7 @@ public class PythonSdkUpdater implements StartupActivity.Background {
private static void updateLocalSdkVersion(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator) {
if (!PythonSdkUtil.isRemote(sdk)) {
final SdkModificator modificatorToRead = sdkModificator != null ? sdkModificator : sdk.getSdkModificator();
ProgressManager.progress(PyBundle.message("sdk.updating.interpreter.version"));
final String versionString = sdk.getSdkType().getVersionString(sdk);
if (!StringUtil.equals(versionString, modificatorToRead.getVersionString())) {
changeSdkModificator(sdk, sdkModificator, modificatorToWrite -> {
@@ -275,22 +416,13 @@ public class PythonSdkUpdater implements StartupActivity.Background {
* <p>
* May be invoked from any thread. May freeze the current thread while evaluating sys.path.
*/
private static boolean updateLocalSdkPaths(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable Project project) {
private static void updateLocalSdkPaths(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable Project project)
throws InvalidSdkException {
if (!PythonSdkUtil.isRemote(sdk)) {
final List<VirtualFile> localSdkPaths;
final boolean forceCommit = ensureBinarySkeletonsDirectoryExists(sdk);
try {
localSdkPaths = getLocalSdkPaths(sdk, project);
}
catch (InvalidSdkException e) {
if (!PythonSdkUtil.isInvalid(sdk)) {
LOG.error(e);
}
return false;
}
final List<VirtualFile> localSdkPaths = getLocalSdkPaths(sdk, project);
commitSdkPathsIfChanged(sdk, sdkModificator, localSdkPaths, forceCommit);
}
return true;
}
/**
@@ -450,6 +582,7 @@ public class PythonSdkUpdater implements StartupActivity.Background {
throw new IllegalArgumentException("Cannot evaluate sys.path for remote Python interpreter " + sdk);
}
final long startTime = System.currentTimeMillis();
ProgressManager.progress(PyBundle.message("sdk.updating.interpreter.paths"));
final List<String> sysPath = PythonSdkType.getSysPath(sdk);
LOG.info("Updating sys.path took " + (System.currentTimeMillis() - startTime) + " ms");
return sysPath;
@@ -512,9 +645,13 @@ public class PythonSdkUpdater implements StartupActivity.Background {
}
private enum Trigger {
STARTUP_ACTIVITY("com.jetbrains.python.sdk.PythonSdkUpdater$1.run"),
STARTUP_ACTIVITY("com.jetbrains.python.sdk.PythonSdkUpdater.runActivity"),
CHANGE_UNDER_INTERPRETER_ROOTS("com.jetbrains.python.packaging.PyPackageManagerImpl.lambda$subscribeToLocalChanges"),
REFRESH_AFTER_PACKAGING_OPERATION("com.jetbrains.python.packaging.PyPackageManagerImpl.lambda$refresh");
REFRESH_AFTER_PACKAGING_OPERATION("com.jetbrains.python.packaging.PyPackageManagerImpl.lambda$refresh"),
NEW_SDK_GENERATION("com.jetbrains.python.sdk.PySdkExtKt.createSdkByGenerateTask"),
CHANGED_SDK_CONFIGURATION("com.jetbrains.python.configuration.PyActiveSdkConfigurable.apply"),
SDK_RELOAD_IN_SETTINGS("com.jetbrains.python.configuration.PythonSdkDetailsDialog.reloadSdk"),
START_SDK_UPDATES_ACTION("com.jetbrains.python.sdk.PyUpdateProjectSdkAction.lambda$actionPerformed");
private static final Logger LOG = Logger.getInstance(Trigger.class);

553
python/src/com/jetbrains/python/sdk/PythonSdkUpdaterOld.java Normal file
View File

@@ -0,0 +1,553 @@
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.sdk;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer;
import com.intellij.execution.ExecutionException;
import com.intellij.notification.NotificationGroup;
import com.intellij.notification.NotificationType;
import com.intellij.openapi.application.Application;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.application.ModalityState;
import com.intellij.openapi.application.TransactionGuard;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.module.Module;
import com.intellij.openapi.module.ModuleManager;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.Task;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.projectRoots.Sdk;
import com.intellij.openapi.projectRoots.SdkAdditionalData;
import com.intellij.openapi.projectRoots.SdkModificator;
import com.intellij.openapi.roots.OrderRootType;
import com.intellij.openapi.startup.StartupActivity;
import com.intellij.openapi.ui.Messages;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.StandardFileSystems;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.PathMappingSettings;
import com.intellij.util.Processor;
import com.intellij.util.concurrency.BlockingSet;
import com.jetbrains.python.PyBundle;
import com.jetbrains.python.codeInsight.typing.PyTypeShed;
import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil;
import com.jetbrains.python.packaging.PyPackageManager;
import com.jetbrains.python.psi.PyUtil;
import com.jetbrains.python.remote.PyRemoteSdkAdditionalDataBase;
import com.jetbrains.python.remote.UnsupportedPythonSdkTypeException;
import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.File;
import java.util.List;
import java.util.*;
import java.util.concurrent.Future;
/**
* Refreshes all project's Python SDKs.
*
* @author vlan
* @author yole
* @noinspection DuplicatedCode
* @deprecated This is the legacy implementation of {@link PythonSdkUpdater}. Don't use it directly.
*/
@Deprecated
public class PythonSdkUpdaterOld implements StartupActivity.Background {
private static final Logger LOG = Logger.getInstance(PythonSdkUpdaterOld.class);
private static final Object ourLock = new Object();
private static final Set<String> ourScheduledToRefresh = new HashSet<String>();
private static final BlockingSet<String> ourUnderRefresh = new BlockingSet<>();
private static final NotificationGroup NOTIFICATION_GROUP = PythonSdkUpdater.NOTIFICATION_GROUP;
/**
* Refreshes the SDKs of the modules for the open project after some delay.
*/
@Override
public void runActivity(@NotNull final Project project) {
updateProjectSdksOnStartup(project);
}
static void updateProjectSdksOnStartup(@NotNull Project project) {
Application application = ApplicationManager.getApplication();
if (application.isUnitTestMode()) return;
if (project.isDisposed()) return;
ProgressManager.getInstance().run(new Task.Backgroundable(project, PyBundle.message("python.sdk.updating.python.paths"), false) {
@Override
public void run(@NotNull ProgressIndicator indicator) {
for (Sdk sdk : getPythonSdks(project)) {
update(sdk, null, project, null);
}
}
});
}
/**
* Updates the paths of an SDK and regenerates its skeletons as a background task.
*
* May be invoked from any thread. May freeze the current thread while evaluating sys.path.
*
* For a local SDK it commits all the SDK paths and runs a background task for updating skeletons. For a remote SDK it runs a background
* task for updating skeletons that saves path mappings in the additional SDK data and then commits all the SDK paths.
*
* The commit of the changes in the SDK happens in the AWT thread while the current thread is waiting the result.
*
* @param sdkModificator if null then it tries to get an SDK modifier from the SDK table, falling back to the modifier of the SDK
* passed as an argument accessed from the AWT thread
* @return false if there was an immediate problem updating the SDK. Other problems are reported as log entries and balloons.
*/
public static boolean update(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable final Project project,
@Nullable final Component ownerComponent) {
final Application application = ApplicationManager.getApplication();
assert !application.isWriteAccessAllowed() : "sdk update should not be run in edt neither under write action";
final String key = PythonSdkType.getSdkKey(sdk);
synchronized (ourLock) {
ourScheduledToRefresh.add(key);
}
String sdkHome = sdk.getHomePath();
if (sdkHome != null && (PythonSdkUtil.isVirtualEnv(sdkHome) || PythonSdkUtil.isConda(sdk))) {
final Future<?> updateSdkFeature = application.executeOnPooledThread(() -> {
PySdkUtil.activateVirtualEnv(sdk); // pre-cache virtualenv activated environment
});
if (ApplicationManager.getApplication().isUnitTestMode()) {
// Running SDK update in background is inappropriate for tests: test may complete before update and updater thread will leak
try {
updateSdkFeature.get();
}
catch (final InterruptedException | java.util.concurrent.ExecutionException e) {
throw new AssertionError("Exception thrown while synchronizing with sdk updater ", e);
}
}
}
updateLocalSdkVersion(sdk, sdkModificator);
if (!updateLocalSdkPaths(sdk, sdkModificator, project)) {
return false;
}
if (project == null) {
return true;
}
if (application.isUnitTestMode()) {
// All actions we take after this line are dedicated to skeleton update process. Not all tests do need them. To find test API that
// updates skeleton, see PySkeletonRefresher
return true;
}
final Throwable methodCallStacktrace = new Throwable("SDK update trace");
application.invokeLater(() -> {
synchronized (ourLock) {
if (!ourScheduledToRefresh.contains(key)) {
if (Trigger.LOG.isDebugEnabled()) {
Trigger.LOG.debug("Dropping simultaneous SDK update for " + key + " triggered by " + Trigger
.getCauseByTrace(methodCallStacktrace));
}
return;
}
ourScheduledToRefresh.remove(key);
}
if (project.isDisposed()) {
return;
}
if (PythonSdkUtil.findSdkByKey(key) == null) {
return;
}
ProgressManager.getInstance().run(new Task.Backgroundable(project, PyBundle.message("sdk.gen.updating.interpreter"), false) {
@Override
public void run(@NotNull ProgressIndicator indicator) {
final Project project1 = getProject();
final Sdk sdkInsideTask = PythonSdkUtil.findSdkByKey(key);
if (sdkInsideTask != null) {
ourUnderRefresh.put(key);
try {
final String skeletonsPath = PythonSdkUtil.getSkeletonsPath(sdk);
try {
if (PythonSdkUtil.isRemote(sdkInsideTask) && project1 == null && ownerComponent == null) {
LOG.error("For refreshing skeletons of remote SDK, either project or owner component must be specified");
}
final String sdkPresentableName = getSdkPresentableName(sdk);
LOG.info("Performing background update of skeletons for SDK " + sdkPresentableName);
indicator.setText(PyBundle.message("python.sdk.updating.skeletons"));
try {
if (Trigger.LOG.isDebugEnabled()) {
Trigger.LOG.debug("Performing skeletons update triggered by " + Trigger.getCauseByTrace(methodCallStacktrace));
}
PySkeletonRefresher.refreshSkeletonsOfSdk(project1, ownerComponent, skeletonsPath, sdkInsideTask);
updateRemoteSdkPaths(sdkInsideTask, getProject());
indicator.setIndeterminate(true);
indicator.setText(PyBundle.message("python.sdk.scanning.installed.packages"));
indicator.setText2("");
LOG.info("Performing background scan of packages for SDK " + sdkPresentableName);
PyPackageManager.getInstance(sdkInsideTask).refreshAndGetPackages(true);
}
catch (ExecutionException e) {
if (LOG.isDebugEnabled()) {
e.initCause(methodCallStacktrace);
LOG.debug(e);
}
else {
LOG.warn(e.getMessage());
}
}
if (project1 != null) {
application.invokeLater(() -> DaemonCodeAnalyzer.getInstance(project1).restart(), project1.getDisposed());
}
}
catch (UnsupportedPythonSdkTypeException e) {
NOTIFICATION_GROUP
.createNotification(PyBundle.message("sdk.gen.failed.notification.title"), null,
PyBundle.message("remote.interpreter.support.is.not.available", sdk.getName()),
NotificationType.WARNING)
.notify(project1);
}
catch (InvalidSdkException e) {
if (PythonSdkUtil.isRemote(sdkInsideTask)) {
PythonSdkType.notifyRemoteSdkSkeletonsFail(e, () -> {
final Sdk sdkInsideNotify = PythonSdkUtil.findSdkByKey(key);
if (sdkInsideNotify != null) {
update(sdkInsideNotify, null, project1, ownerComponent);
}
});
}
else if (!PythonSdkUtil.isInvalid(sdkInsideTask)) {
LOG.error(e);
}
}
}
finally {
try {
ourUnderRefresh.remove(key);
}
catch (IllegalStateException e) {
LOG.error(e);
}
}
}
}
});
}, ModalityState.NON_MODAL);
return true;
}
/**
* Updates the paths of an SDK and regenerates its skeletons as a background task. Shows an error message if the update fails.
*
* @see #update(Sdk, SdkModificator, Project, Component)
*/
public static void updateOrShowError(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable Project project,
@Nullable Component ownerComponent) {
final boolean success = update(sdk, sdkModificator, project, ownerComponent);
if (!success) {
Messages.showErrorDialog(project,
PyBundle.message("MSG.cant.setup.sdk.$0", getSdkPresentableName(sdk)),
PyBundle.message("MSG.title.bad.sdk"));
}
}
/**
* Changes the version string of an SDK if it's out of date.
* <p>
* May be invoked from any thread. May freeze the current thread while evaluating the run-time Python version.
*/
private static void updateLocalSdkVersion(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator) {
if (!PythonSdkUtil.isRemote(sdk)) {
final SdkModificator modificatorToRead = sdkModificator != null ? sdkModificator : sdk.getSdkModificator();
final String versionString = sdk.getSdkType().getVersionString(sdk);
if (!StringUtil.equals(versionString, modificatorToRead.getVersionString())) {
changeSdkModificator(sdk, sdkModificator, modificatorToWrite -> {
modificatorToWrite.setVersionString(versionString);
return true;
});
}
}
}
/**
* Updates the paths of a local SDK.
* <p>
* May be invoked from any thread. May freeze the current thread while evaluating sys.path.
*/
private static boolean updateLocalSdkPaths(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable Project project) {
if (!PythonSdkUtil.isRemote(sdk)) {
final List<VirtualFile> localSdkPaths;
final boolean forceCommit = ensureBinarySkeletonsDirectoryExists(sdk);
try {
localSdkPaths = getLocalSdkPaths(sdk, project);
}
catch (InvalidSdkException e) {
if (!PythonSdkUtil.isInvalid(sdk)) {
LOG.error(e);
}
return false;
}
commitSdkPathsIfChanged(sdk, sdkModificator, localSdkPaths, forceCommit);
}
return true;
}
/**
* Updates the paths of a remote SDK.
* <p>
* Requires the skeletons refresh steps to be run before it in order to get remote paths mappings in the additional SDK data.
* <p>
* You may invoke it from any thread. Blocks until the commit is done in the AWT thread.
*/
private static void updateRemoteSdkPaths(@NotNull Sdk sdk, @Nullable Project project) {
if (PythonSdkUtil.isRemote(sdk)) {
final boolean forceCommit = ensureBinarySkeletonsDirectoryExists(sdk);
final List<VirtualFile> remoteSdkPaths = getRemoteSdkPaths(sdk, project);
commitSdkPathsIfChanged(sdk, null, remoteSdkPaths, forceCommit);
}
}
private static boolean ensureBinarySkeletonsDirectoryExists(Sdk sdk) {
final String skeletonsPath = PythonSdkUtil.getSkeletonsPath(sdk);
if (skeletonsPath != null) {
if (new File(skeletonsPath).mkdirs()) {
return true;
}
}
return false;
}
/**
* Returns all the paths for a local SDK.
*/
@NotNull
private static List<VirtualFile> getLocalSdkPaths(@NotNull Sdk sdk, @Nullable Project project) throws InvalidSdkException {
return ImmutableList.<VirtualFile>builder()
.addAll(filterRootPaths(sdk, evaluateSysPath(sdk), project))
.addAll(getSkeletonsPaths(sdk))
.addAll(getUserAddedPaths(sdk))
.addAll(PyTypeShed.INSTANCE.findRootsForSdk(sdk))
.build();
}
/**
* Returns all the paths for a remote SDK.
* <p>
* Requires the skeletons refresh steps to be run before it in order to get remote paths mappings in the additional SDK data.
*/
@NotNull
private static List<VirtualFile> getRemoteSdkPaths(@NotNull Sdk sdk, @Nullable Project project) {
return ImmutableList.<VirtualFile>builder()
.addAll(getRemoteSdkMappedPaths(sdk, project))
.addAll(getSkeletonsPaths(sdk))
.addAll(getUserAddedPaths(sdk))
.addAll(PyTypeShed.INSTANCE.findRootsForSdk(sdk))
.build();
}
/**
* Returns all the paths manually added to an SDK by the user.
*/
@NotNull
private static List<VirtualFile> getUserAddedPaths(@NotNull Sdk sdk) {
final SdkAdditionalData additionalData = sdk.getSdkAdditionalData();
final PythonSdkAdditionalData pythonAdditionalData = PyUtil.as(additionalData, PythonSdkAdditionalData.class);
return pythonAdditionalData != null ? Lists.newArrayList(pythonAdditionalData.getAddedPathFiles()) :
Collections.emptyList();
}
/**
* Returns local paths for a remote SDK that have been mapped to remote paths during the skeleton refresh step.
* <p>
* Returns all the existing paths except those manually excluded by the user.
*/
@NotNull
private static List<VirtualFile> getRemoteSdkMappedPaths(@NotNull Sdk sdk, @Nullable Project project) {
final SdkAdditionalData additionalData = sdk.getSdkAdditionalData();
if (additionalData instanceof PyRemoteSdkAdditionalDataBase) {
final PyRemoteSdkAdditionalDataBase remoteSdkData = (PyRemoteSdkAdditionalDataBase)additionalData;
final List<String> paths = new ArrayList<>();
for (PathMappingSettings.PathMapping mapping : remoteSdkData.getPathMappings().getPathMappings()) {
paths.add(mapping.getLocalRoot());
}
return filterRootPaths(sdk, paths, project);
}
return Collections.emptyList();
}
/**
* Filters valid paths from an initial set of Python paths and returns them as virtual files.
*/
@NotNull
public static List<VirtualFile> filterRootPaths(@NotNull Sdk sdk, @NotNull List<String> paths, @Nullable Project project) {
final PythonSdkAdditionalData pythonAdditionalData = PyUtil.as(sdk.getSdkAdditionalData(), PythonSdkAdditionalData.class);
final Collection<VirtualFile> excludedPaths = pythonAdditionalData != null ? pythonAdditionalData.getExcludedPathFiles() :
Collections.emptyList();
final Set<VirtualFile> moduleRoots = new HashSet<>();
if (project != null) {
final Module[] modules = ModuleManager.getInstance(project).getModules();
for (Module module : modules) {
moduleRoots.addAll(PyUtil.getSourceRoots(module));
}
}
final List<VirtualFile> results = new ArrayList<>();
// TODO: Refactor SDK so they can provide exclusions for root paths
final VirtualFile condaFolder = PythonSdkUtil.isConda(sdk) ? PythonSdkUtil.getCondaDirectory(sdk) : null;
for (String path : paths) {
if (path != null && !FileUtilRt.extensionEquals(path, "egg-info")) {
final VirtualFile virtualFile = StandardFileSystems.local().refreshAndFindFileByPath(path);
if (virtualFile != null && !virtualFile.equals(condaFolder)) {
final VirtualFile rootFile = PythonSdkType.getSdkRootVirtualFile(virtualFile);
if (!excludedPaths.contains(rootFile) && !moduleRoots.contains(rootFile)) {
results.add(rootFile);
continue;
}
}
}
LOG.info("Bogus sys.path entry " + path);
}
return results;
}
/**
* Returns the paths of the binary skeletons and user skeletons for an SDK.
*/
@NotNull
private static List<VirtualFile> getSkeletonsPaths(@NotNull Sdk sdk) {
final List<VirtualFile> results = new ArrayList<>();
final String skeletonsPath = PythonSdkUtil.getSkeletonsPath(sdk);
if (skeletonsPath != null) {
final VirtualFile skeletonsDir = StandardFileSystems.local().refreshAndFindFileByPath(skeletonsPath);
if (skeletonsDir != null) {
results.add(skeletonsDir);
LOG.info("Binary skeletons directory for SDK " + getSdkPresentableName(sdk) + "): " + skeletonsDir.getPath());
}
}
final VirtualFile userSkeletonsDir = PyUserSkeletonsUtil.getUserSkeletonsDirectory();
if (userSkeletonsDir != null) {
results.add(userSkeletonsDir);
LOG.info("User skeletons directory for SDK " + getSdkPresentableName(sdk) + "): " + userSkeletonsDir.getPath());
}
return results;
}
@NotNull
private static String getSdkPresentableName(@NotNull Sdk sdk) {
final String homePath = sdk.getHomePath();
final String name = sdk.getName();
return homePath != null ? name + " (" + homePath + ")" : name;
}
/**
* Evaluates sys.path by running the Python interpreter from a local SDK.
* <p>
* Returns all the existing paths except those manually excluded by the user.
*/
@NotNull
private static List<String> evaluateSysPath(@NotNull Sdk sdk) throws InvalidSdkException {
if (PythonSdkUtil.isRemote(sdk)) {
throw new IllegalArgumentException("Cannot evaluate sys.path for remote Python interpreter " + sdk);
}
final long startTime = System.currentTimeMillis();
final List<String> sysPath = PythonSdkType.getSysPath(sdk);
LOG.info("Updating sys.path took " + (System.currentTimeMillis() - startTime) + " ms");
return sysPath;
}
/**
* Commits new SDK paths using an SDK modificator if the paths have been changed.
* <p>
* You may invoke it from any thread. Blocks until the commit is done in the AWT thread.
*/
private static void commitSdkPathsIfChanged(@NotNull Sdk sdk,
@Nullable final SdkModificator sdkModificator,
@NotNull final List<VirtualFile> sdkPaths,
boolean forceCommit) {
final SdkModificator modificatorToRead = sdkModificator != null ? sdkModificator : sdk.getSdkModificator();
final List<VirtualFile> currentSdkPaths = Arrays.asList(modificatorToRead.getRoots(OrderRootType.CLASSES));
if (forceCommit || !Sets.newHashSet(sdkPaths).equals(Sets.newHashSet(currentSdkPaths))) {
changeSdkModificator(sdk, sdkModificator, effectiveModificator -> {
effectiveModificator.removeAllRoots();
for (VirtualFile sdkPath : sdkPaths) {
effectiveModificator.addRoot(PythonSdkType.getSdkRootVirtualFile(sdkPath), OrderRootType.CLASSES);
}
return true;
});
}
}
/**
* Applies a processor to an SDK modificator or an SDK and commits it.
* <p>
* You may invoke it from any threads. Blocks until the commit is done in the AWT thread.
*/
private static void changeSdkModificator(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator,
@NotNull Processor<SdkModificator> processor) {
final String key = PythonSdkType.getSdkKey(sdk);
TransactionGuard.getInstance().assertWriteSafeContext(ModalityState.defaultModalityState());
ApplicationManager.getApplication().invokeAndWait(() -> {
final Sdk sdkInsideInvoke = PythonSdkUtil.findSdkByKey(key);
final SdkModificator effectiveModificator = sdkModificator != null ? sdkModificator :
sdkInsideInvoke != null ? sdkInsideInvoke.getSdkModificator() : sdk.getSdkModificator();
if (processor.process(effectiveModificator)) {
effectiveModificator.commitChanges();
}
});
}
/**
* Returns unique Python SDKs for the open modules of the project.
*/
@NotNull
private static Set<Sdk> getPythonSdks(@NotNull Project project) {
final Set<Sdk> pythonSdks = new LinkedHashSet<Sdk>();
for (Module module : ModuleManager.getInstance(project).getModules()) {
final Sdk sdk = PythonSdkUtil.findPythonSdk(module);
if (sdk != null && sdk.getSdkType() instanceof PythonSdkType) {
pythonSdks.add(sdk);
}
}
return pythonSdks;
}
private enum Trigger {
STARTUP_ACTIVITY("com.jetbrains.python.sdk.PythonSdkUpdater$1.run"),
CHANGE_UNDER_INTERPRETER_ROOTS("com.jetbrains.python.packaging.PyPackageManagerImpl.lambda$subscribeToLocalChanges"),
REFRESH_AFTER_PACKAGING_OPERATION("com.jetbrains.python.packaging.PyPackageManagerImpl.lambda$refresh"),
START_SDK_UPDATES_ACTION("com.jetbrains.python.sdk.PyUpdateProjectSdkAction.actionPerformed");
private static final Logger LOG = Logger.getInstance(Trigger.class);
private final String myFrameMarker;
Trigger(@NotNull String frameMarker) {
myFrameMarker = frameMarker;
}
@NotNull
public static String getCauseByTrace(@NotNull Throwable trace) {
final Trigger trigger = findTriggerByTrace(trace);
if (trigger != null) {
return trigger.name();
}
return "Unknown trigger:\n" + ExceptionUtil.getThrowableText(trace);
}
@Nullable
public static Trigger findTriggerByTrace(@NotNull Throwable trace) {
final String traceText = ExceptionUtil.getThrowableText(trace);
for (Trigger value : values()) {
if (traceText.contains(value.myFrameMarker)) {
return value;
}
}
return null;
}
}}