diff --git a/platform/util/src/com/intellij/util/concurrency/BlockingSet.java b/platform/util/src/com/intellij/util/concurrency/BlockingSet.java new file mode 100644 index 000000000000..2833036904d2 --- /dev/null +++ b/platform/util/src/com/intellij/util/concurrency/BlockingSet.java @@ -0,0 +1,84 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.concurrency; + +import com.intellij.util.containers.HashSet; +import org.jetbrains.annotations.NotNull; + +import java.util.Set; +import java.util.concurrent.locks.*; + +/** + * Naive implementation of blocking set: class that allows {@link #put(Object)} lock by the {@code key} specified and + * {@link #remove(Object)} it. The main feature is that another invocation of {@link #put(Object)} with the same {@code key} could not + * be proceeded while one won't call {@link #remove(Object)} with this {@code key}. + * + * @author Alexander Koshevoy + */ +public class BlockingSet { + private final Set set; + + private final Condition unlock; + private final Lock lock; + + public BlockingSet() { + set = new HashSet(); + lock = new ReentrantLock(); + unlock = lock.newCondition(); + } + + /** + * Acquires lock by {@code key}. If lock by {@code key} has been already acquired wait until it is released. Acquire is not reentrant. + * + * @param key + */ + public void put(@NotNull T key) { + lock.lock(); + try { + while (set.contains(key)) { + unlock.awaitUninterruptibly(); + } + set.add(key); + } + finally { + lock.unlock(); + } + } + + /** + * Releases lock by {@code key}. If lock has not been acquired throws {@link IllegalStateException}. + * + * @param key + * @throws IllegalStateException if lock by {@code key} has not been acquired. + */ + public void remove(@NotNull T key) throws IllegalStateException { + lock.lock(); + try { + if (!set.contains(key)) { + throw new IllegalStateException(); + } + set.remove(key); + unlock.signalAll(); + } + finally { + lock.unlock(); + } + } + + public static BlockingSet newInstance() { + return new BlockingSet(); + } +} diff --git a/platform/util/testSrc/com/intellij/util/concurrency/BlockingSetTest.java b/platform/util/testSrc/com/intellij/util/concurrency/BlockingSetTest.java new file mode 100644 index 000000000000..3d1273c2e443 --- /dev/null +++ b/platform/util/testSrc/com/intellij/util/concurrency/BlockingSetTest.java @@ -0,0 +1,68 @@ +package com.intellij.util.concurrency; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author Alexander Koshevoy + */ +public class BlockingSetTest { + @Test + public void testSingleThreadLock() throws Exception { + BlockingSet lock = new BlockingSet(); + lock.put("eins"); + lock.put("zwei"); + lock.remove("zwei"); + lock.put("polizei"); + lock.remove("polizei"); + lock.remove("eins"); + } + + @Test(expected = IllegalStateException.class) + public void testReleaseNotAcquired() throws Exception { + BlockingSet lock = new BlockingSet(); + lock.put("eins"); + lock.put("zwei"); + lock.remove("eins"); + lock.remove("polizei"); + } + + @Test + public void testMultipleThreads() throws Exception { + final BlockingSet lock = new BlockingSet(); + int threads = 10; + int tasks = 100; + ExecutorService service = Executors.newFixedThreadPool(threads); + List> taskList = new ArrayList>(tasks); + final AtomicBoolean check = new AtomicBoolean(false); + for (int i = 0; i < tasks; i++) { + taskList.add(new Callable() { + @Override + public Void call() throws Exception { + lock.put("key"); + try { + Assert.assertFalse(check.get()); + check.set(true); + Thread.sleep(10L); + check.set(false); + } + finally { + lock.remove("key"); + } + return null; + } + }); + } + List> futures = service.invokeAll(taskList); + service.shutdown(); + for (Future future : futures) { + future.get(); + } + Assert.assertTrue(service.awaitTermination(100, TimeUnit.SECONDS)); + } +} \ No newline at end of file diff --git a/python/ide/src/com/jetbrains/python/configuration/PyActiveSdkConfigurable.java b/python/ide/src/com/jetbrains/python/configuration/PyActiveSdkConfigurable.java index 6adda56d1dd5..f8124626d520 100644 --- a/python/ide/src/com/jetbrains/python/configuration/PyActiveSdkConfigurable.java +++ b/python/ide/src/com/jetbrains/python/configuration/PyActiveSdkConfigurable.java @@ -268,15 +268,8 @@ public class PyActiveSdkConfigurable implements UnnamedConfigurable { myProjectSdksModel.addSdk(selectedSdk); } } - else if (myInitialSdkSet.contains(selectedSdk)) { - final Sdk finalSelectedSdk = selectedSdk; - ApplicationManager.getApplication().invokeLater(new Runnable() { - @Override - public void run() { - assert finalSelectedSdk != null; - PythonSdkType.getInstance().setupSdkPaths(finalSelectedSdk, myProject, null); - } - }); + else if (myInitialSdkSet.contains(selectedSdk) && selectedSdk != null) { + PythonSdkUpdater.updateOrShowError(selectedSdk, null, myProject, null); } if (selectedSdk != null) { updateSdkList(false); diff --git a/python/ide/src/com/jetbrains/python/configuration/PythonSdkDetailsDialog.java b/python/ide/src/com/jetbrains/python/configuration/PythonSdkDetailsDialog.java index cf233554b20e..cb00c0594eb7 100644 --- a/python/ide/src/com/jetbrains/python/configuration/PythonSdkDetailsDialog.java +++ b/python/ide/src/com/jetbrains/python/configuration/PythonSdkDetailsDialog.java @@ -57,6 +57,7 @@ import com.jetbrains.python.remote.PyRemoteSourceItem; import com.jetbrains.python.remote.PythonRemoteInterpreterManager; import com.jetbrains.python.sdk.*; import com.jetbrains.python.sdk.flavors.PythonSdkFlavor; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; @@ -429,9 +430,8 @@ public class PythonSdkDetailsDialog extends DialogWrapper { } } - private void reloadSdk(Sdk currentSdk) { - PythonSdkType.getInstance() - .setupSdkPaths(currentSdk, myProject, null, myModificators.get(currentSdk)); // or must it be a RunWriteAction? + private void reloadSdk(@NotNull Sdk currentSdk) { + PythonSdkUpdater.update(currentSdk, myModificators.get(currentSdk), myProject, null); } private class ToggleVirtualEnvFilterButton extends ToggleActionButton implements DumbAware { diff --git a/python/ide/src/com/jetbrains/python/newProject/actions/GenerateProjectCallback.java b/python/ide/src/com/jetbrains/python/newProject/actions/GenerateProjectCallback.java index 5426d9864157..ee5498510bb6 100644 --- a/python/ide/src/com/jetbrains/python/newProject/actions/GenerateProjectCallback.java +++ b/python/ide/src/com/jetbrains/python/newProject/actions/GenerateProjectCallback.java @@ -36,10 +36,7 @@ import com.intellij.util.NullableConsumer; import com.jetbrains.python.configuration.PyConfigurableInterpreterList; import com.jetbrains.python.newProject.PyNewProjectSettings; import com.jetbrains.python.newProject.PythonProjectGenerator; -import com.jetbrains.python.sdk.PyDetectedSdk; -import com.jetbrains.python.sdk.PySdkService; -import com.jetbrains.python.sdk.PythonSdkAdditionalData; -import com.jetbrains.python.sdk.PythonSdkType; +import com.jetbrains.python.sdk.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -68,7 +65,7 @@ public class GenerateProjectCallback implements NullableConsumerThese will be fully re-generated, which will take some time, but will happen only once.
Next time you open the project, only skeletons of new or updated binary modules will be re-generated. -sdk.gen.updating.skels=Updating Skeletons +sdk.gen.updating.skeletons=Updating Skeletons sdk.gen.stubs.for.binary.modules=Generate stubs for binary module {0} # Active SDK configurable and related dialogs diff --git a/python/src/com/jetbrains/python/codeInsight/userSkeletons/PyUserSkeletonsUtil.java b/python/src/com/jetbrains/python/codeInsight/userSkeletons/PyUserSkeletonsUtil.java index 619130fa8eeb..32080cb67c91 100644 --- a/python/src/com/jetbrains/python/codeInsight/userSkeletons/PyUserSkeletonsUtil.java +++ b/python/src/com/jetbrains/python/codeInsight/userSkeletons/PyUserSkeletonsUtil.java @@ -20,7 +20,6 @@ import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.extensions.Extensions; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; -import com.intellij.openapi.roots.OrderRootType; import com.intellij.openapi.util.Key; import com.intellij.openapi.vfs.StandardFileSystems; import com.intellij.openapi.vfs.VfsUtilCore; @@ -38,7 +37,6 @@ import com.jetbrains.python.psi.resolve.*; import com.jetbrains.python.psi.types.PyClassLikeType; import com.jetbrains.python.psi.types.PyType; import com.jetbrains.python.psi.types.TypeEvalContext; -import com.jetbrains.python.sdk.PySdkUpdater; import com.jetbrains.python.sdk.PythonSdkType; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -148,13 +146,6 @@ public class PyUserSkeletonsUtil { return null; } - public static void addUserSkeletonsRoot(@NotNull PySdkUpdater sdkUpdater) { - final VirtualFile root = getUserSkeletonsDirectory(); - if (root != null) { - sdkUpdater.addRoot(root, OrderRootType.CLASSES); - } - } - @Nullable private static PsiElement getUserSkeleton(@NotNull PyElement element, @NotNull PyFile skeletonFile, @Nullable TypeEvalContext context) { if (element instanceof PyFile) { diff --git a/python/src/com/jetbrains/python/sdk/PySdkUpdater.java b/python/src/com/jetbrains/python/sdk/PySdkUpdater.java deleted file mode 100644 index c545c3fcb207..000000000000 --- a/python/src/com/jetbrains/python/sdk/PySdkUpdater.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2000-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jetbrains.python.sdk; - -import com.google.common.collect.Maps; -import com.intellij.openapi.application.ApplicationManager; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.projectRoots.Sdk; -import com.intellij.openapi.projectRoots.SdkModificator; -import com.intellij.openapi.roots.OrderRootType; -import com.intellij.openapi.vfs.VirtualFile; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.Map; - -/** - * This sdk updater class is a facade to Sdk to make changes in it in a reliable way. - * Working with sdk instance instead of this class can be wrong, because an instance can become - * obsolete being substituted in sdk table by a new one. Or already created sdk modificator can be committed, - * discarding the changes that we doing with the sdk. - *

- *

- * There are two ways of creation of the facade: - * 1) by sdk path - in this case we'll get the current actual sdk instance - * from sdk table by that path, creating and committing SdkModificator on every change - * 2) by sdkModificator - in that case we'll make changes to that modificator, but it is not committed, - * because it has been created outside of the updater. - * - * @author traff - */ -public abstract class PySdkUpdater { - private static final Logger LOG = Logger.getInstance("#" + PySdkUpdater.class.getName()); - - private static Map mySdkUpdaters = Maps.newHashMap(); - - @NotNull - public abstract Sdk getSdk(); - - public abstract void modifySdk(@NotNull SdkModificationProcessor processor); - - public void addRoot(@NotNull final VirtualFile root, @NotNull final OrderRootType rootType) { - modifySdk(new SdkModificationProcessor() { - @Override - public void process(@NotNull Sdk sdk, @NotNull SdkModificator sdkModificator) { - sdkModificator.addRoot(root, rootType); - } - }); - } - - public void removeRoots(@NotNull final OrderRootType rootType) { - modifySdk(new PySdkUpdater.SdkModificationProcessor() { - @Override - public void process(@NotNull Sdk sdk, @NotNull SdkModificator sdkModificator) { - sdkModificator.removeRoots(rootType); - } - }); - } - - public abstract void commit(); - - public static synchronized PySdkUpdater fromSdkPath(@Nullable String sdkPath) { - checkSingleton(sdkPath); - return new JdkTableUpdater(sdkPath); - } - - public static synchronized PySdkUpdater fromSdkModificator(@NotNull Sdk sdk, @NotNull SdkModificator sdkModificator) { - checkSingleton(sdk.getHomePath()); - return new SdkModificatorUpdater(sdk, sdkModificator); - } - - public static synchronized PySdkUpdater singletonJdkTableUpdater(@NotNull String sdkPath) { - if (mySdkUpdaters.get(sdkPath) == null) { - Sdk sdk = PythonSdkType.findSdkByPath(sdkPath); - if (sdk != null) { - mySdkUpdaters.put(sdkPath, new SingletonSdkModificatorUpdater(sdk, sdk.getSdkModificator())); - } else { - throw new PySdkNotFoundException(); - } - } - return mySdkUpdaters.get(sdkPath); - } - - - private static synchronized void checkSingleton(String sdkPath) { - if (mySdkUpdaters.get(sdkPath) != null) { - LOG.error("Changing more then one sdkModificator at a time"); - } - } - - private static synchronized void checkNotDisposed(String sdkPath) { - if (mySdkUpdaters.get(sdkPath) == null) { - LOG.error("sdk modificator is already committed, further changes will be discarded"); - } - } - - private static synchronized void disposeSingleton(String sdkPath) { - mySdkUpdaters.put(sdkPath, null); - } - - @Nullable - public abstract String getHomePath(); - - public abstract VirtualFile[] getRoots(OrderRootType rootType); - - private static class JdkTableUpdater extends PySdkUpdater { - private final String mySdkPath; - - private JdkTableUpdater(@Nullable String path) { - mySdkPath = path; - } - - @NotNull - @Override - public Sdk getSdk() { - Sdk sdk = PythonSdkType.findSdkByPath(mySdkPath); - if (sdk != null) { - return sdk; - } - else { - throw new PySdkNotFoundException(); - } - } - - @Nullable - @Override - public String getHomePath() { - return mySdkPath; - } - - @Override - public VirtualFile[] getRoots(OrderRootType rootType) { - return getSdk().getRootProvider().getFiles(rootType); - } - - public void modifySdk(@NotNull SdkModificationProcessor processor) { - checkSingleton(mySdkPath); - - ApplicationManager.getApplication().assertIsDispatchThread(); - - Sdk sdk = PythonSdkType.findSdkByPath(mySdkPath); - - if (sdk != null) { - SdkModificator modificator = getSdk().getSdkModificator(); - processor.process(sdk, modificator); - modificator.commitChanges(); - } - } - - @Override - public void commit() { - // all changes are already committed - } - } - - private static class SdkModificatorUpdater extends PySdkUpdater { - private SdkModificator myModificator; - private Sdk mySdk; - - public SdkModificatorUpdater(@NotNull Sdk sdk, @NotNull SdkModificator modificator) { - mySdk = sdk; - myModificator = modificator; - } - - @NotNull - @Override - public Sdk getSdk() { - return mySdk; - } - - @NotNull - @Override - public String getHomePath() { - return mySdk.getHomePath(); - } - - @Override - public VirtualFile[] getRoots(OrderRootType rootType) { - return myModificator.getRoots(rootType); - } - - public void modifySdk(@NotNull SdkModificationProcessor processor) { - processor.process(getSdk(), myModificator); - } - - @Override - public void commit() { - checkSingleton(getHomePath()); - myModificator.commitChanges(); - } - } - - private static class SingletonSdkModificatorUpdater extends SdkModificatorUpdater { - public SingletonSdkModificatorUpdater(@NotNull Sdk sdk, @NotNull SdkModificator modificator) { - super(sdk, modificator); - } - - @Override - public void modifySdk(@NotNull SdkModificationProcessor processor) { - checkNotDisposed(getHomePath()); - super.modifySdk(processor); - } - - @Override - public void commit() { - disposeSingleton(getHomePath()); - super.commit(); - } - } - - - public interface SdkModificationProcessor { - void process(@NotNull Sdk sdk, @NotNull SdkModificator sdkModificator); - } - - public static class PySdkNotFoundException extends RuntimeException { - } -} diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkType.java b/python/src/com/jetbrains/python/sdk/PythonSdkType.java index 392ab15d39c7..21411bae2937 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkType.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkType.java @@ -31,23 +31,18 @@ import com.intellij.notification.Notifications; import com.intellij.openapi.actionSystem.CommonDataKeys; import com.intellij.openapi.application.Application; import com.intellij.openapi.application.ApplicationManager; -import com.intellij.openapi.application.ModalityState; -import com.intellij.openapi.application.PathManager; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.fileChooser.FileChooserDescriptor; import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleUtilCore; -import com.intellij.openapi.progress.ProgressIndicator; -import com.intellij.openapi.progress.ProgressManager; -import com.intellij.openapi.progress.Task; -import com.intellij.openapi.project.DumbModePermission; -import com.intellij.openapi.project.DumbService; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.*; import com.intellij.openapi.roots.ModuleRootManager; import com.intellij.openapi.roots.OrderRootType; -import com.intellij.openapi.ui.Messages; -import com.intellij.openapi.util.*; +import com.intellij.openapi.util.Comparing; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.Ref; +import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileSystemUtil; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.CharFilter; @@ -64,12 +59,10 @@ import com.intellij.util.Consumer; import com.intellij.util.ExceptionUtil; import com.intellij.util.NullableConsumer; import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.ui.UIUtil; import com.jetbrains.python.PyBundle; import com.jetbrains.python.PyNames; import com.jetbrains.python.PythonFileType; import com.jetbrains.python.PythonHelper; -import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil; import com.jetbrains.python.facet.PythonFacetSettings; import com.jetbrains.python.packaging.PyCondaPackageManagerImpl; import com.jetbrains.python.psi.LanguageLevel; @@ -93,7 +86,6 @@ import java.io.IOException; import java.lang.ref.WeakReference; import java.util.*; import java.util.List; -import java.util.regex.Pattern; /** * @author yole @@ -111,8 +103,6 @@ public class PythonSdkType extends SdkType { private static final Key> SDK_CREATOR_COMPONENT_KEY = Key.create("#com.jetbrains.python.sdk.creatorComponent"); - private Set scheduledToRefresh = ContainerUtil.newConcurrentSet(); - public static PythonSdkType getInstance() { return SdkType.findInstance(PythonSdkType.class); } @@ -505,7 +495,7 @@ public class PythonSdkType extends SdkType { return FileUtil.toSystemDependentName(path); } - public void setupSdkPaths(@NotNull final Sdk sdk) { + public void setupSdkPaths(@NotNull Sdk sdk) { final Project project; final WeakReference ownerComponentRef = sdk.getUserData(SDK_CREATOR_COMPONENT_KEY); final Component ownerComponent = SoftReference.dereference(ownerComponentRef); @@ -515,13 +505,7 @@ public class PythonSdkType extends SdkType { else { project = CommonDataKeys.PROJECT.getData(DataManager.getInstance().getDataContext()); } - - DumbService.allowStartingDumbModeInside(DumbModePermission.MAY_START_BACKGROUND, new Runnable() { - @Override - public void run() { - setupSdkPaths(sdk, project, ownerComponent); - } - }); + PythonSdkUpdater.updateOrShowError(sdk, null, project, ownerComponent); } @Override @@ -529,131 +513,6 @@ public class PythonSdkType extends SdkType { return true; // run setupSdkPaths only once (from PythonSdkDetailsStep). Skip this from showCustomCreateUI } - public void setupSdkPaths(@NotNull final Sdk sdk, - @Nullable final Project project, - @Nullable final Component ownerComponent, - @NotNull final SdkModificator sdkModificator) { - scheduledToRefresh.add(sdk.getHomePath()); - doSetupSdkPaths(project, ownerComponent, PySdkUpdater.fromSdkModificator(sdk, sdkModificator)); - } - - - public void setupSdkPaths(final Sdk sdk, @Nullable final Project project, @Nullable final Component ownerComponent) { - scheduledToRefresh.add(sdk.getHomePath()); - ApplicationManager.getApplication().invokeLater(new Runnable() { - @Override - public void run() { - try { - PySdkUpdater updater = PySdkUpdater.singletonJdkTableUpdater(sdk.getHomePath()); - final boolean success = doSetupSdkPaths(project, ownerComponent, updater); - - if (!success) { - Messages.showErrorDialog( - project, - PyBundle.message("MSG.cant.setup.sdk.$0", FileUtil.toSystemDependentName(sdk.getSdkModificator().getHomePath())), - PyBundle.message("MSG.title.bad.sdk") - ); - } - } - catch (PySdkUpdater.PySdkNotFoundException e) { - // sdk was removed from sdk table so no need to setup paths - } - } - }, ModalityState.NON_MODAL); - } - - public void setupSdkPathsImmediately(final Sdk sdk, @Nullable final Project project) { - scheduledToRefresh.add(sdk.getHomePath()); - final boolean success = doSetupSdkPaths(project, null, PySdkUpdater.fromSdkPath(sdk.getHomePath())); - - if (!success) { - Messages.showErrorDialog( - project, - PyBundle.message("MSG.cant.setup.sdk.$0", FileUtil.toSystemDependentName(sdk.getSdkModificator().getHomePath())), - PyBundle.message("MSG.title.bad.sdk") - ); - } - } - - private boolean doSetupSdkPaths(@Nullable final Project project, - @Nullable final Component ownerComponent, - @NotNull final PySdkUpdater sdkUpdater) { - if (isRemote(sdkUpdater.getSdk()) && project == null && ownerComponent == null) { - LOG.error("For refreshing skeletons of remote SDK, either project or owner component must be specified"); - } - final ProgressManager progressManager = ProgressManager.getInstance(); - boolean sdkPathsUpdated = UIUtil.invokeAndWaitIfNeeded( - new Computable() { - @Override - public Boolean compute() { - return updateSdkPaths(sdkUpdater); - } - } - ); - - final Application application = ApplicationManager.getApplication(); - if (sdkPathsUpdated && !application.isUnitTestMode()) { - application.invokeLater(new Runnable() { - @Override - public void run() { - if (!scheduledToRefresh.contains(sdkUpdater.getHomePath())) { - return; - } - scheduledToRefresh.remove(sdkUpdater.getHomePath()); - progressManager.run(new Task.Backgroundable(project, PyBundle.message("sdk.gen.updating.skels"), false) { - @Override - public void run(@NotNull ProgressIndicator indicator) { - try { - PythonSdkUpdater.updateSdk(project, ownerComponent, sdkUpdater); - } - catch (InvalidSdkException e) { - // If the SDK is invalid, the user should worry about the SDK itself, not about skeletons generation errors - if (isVagrant(sdkUpdater.getSdk()) || isDocker(sdkUpdater.getSdk())) { - notifyRemoteSdkSkeletonsFail(e, new Runnable() { - @Override - public void run() { - setupSdkPaths(sdkUpdater.getSdk(), project, ownerComponent); - } - }); - } - else if (!isInvalid(sdkUpdater.getSdk())) { - LOG.error(e); - } - } finally { - UIUtil.invokeAndWaitIfNeeded(new Runnable() { - @Override - public void run() { - sdkUpdater.commit(); - } - }); - } - } - }); - } - }); - } - return sdkPathsUpdated; - } - - @NotNull - public static Boolean updateSdkPaths(@NotNull PySdkUpdater sdkUpdater) { - sdkUpdater.modifySdk(new PySdkUpdater.SdkModificationProcessor() { - @Override - public void process(@NotNull Sdk sdk, - @NotNull SdkModificator sdkModificator) { - sdkModificator.removeAllRoots(); - } - }); - try { - updateSdkRootsFromSysPath(sdkUpdater); - updateUserAddedPaths(sdkUpdater); - return true; - } - catch (InvalidSdkException ignored) { - } - return false; - } - public static void notifyRemoteSdkSkeletonsFail(final InvalidSdkException e, @Nullable final Runnable restartAction) { NotificationListener notificationListener; String notificationMessage; @@ -718,95 +577,6 @@ public class PythonSdkType extends SdkType { */ public static final OrderRootType BUILTIN_ROOT_TYPE = OrderRootType.CLASSES; - private final static Pattern PYTHON_NN_RE = Pattern.compile("python\\d\\.\\d.*"); - - public static void updateSdkRootsFromSysPath(PySdkUpdater sdkUpdater) - throws InvalidSdkException { - Application application = ApplicationManager.getApplication(); - boolean not_in_unit_test_mode = (application != null && !application.isUnitTestMode()); - - String sdkHome = sdkUpdater.getHomePath(); - assert sdkHome != null; - final String sep = File.separator; - // Add folders from sys.path - if (!PySdkUtil.isRemote(sdkUpdater.getSdk())) { //no sense to add roots of remote sdk - final List paths = getSysPath(sdkHome); - if (paths.size() > 0) { - // add every path as root. - for (String path : paths) { - if (!path.contains(sep)) continue; // TODO: interpret possible 'special' paths reasonably - addSdkRoot(sdkUpdater, path); - } - } - } - - PyUserSkeletonsUtil.addUserSkeletonsRoot(sdkUpdater); - addSkeletonsRoot(sdkUpdater, sdkHome); - - if (not_in_unit_test_mode) { - File venv_root = getVirtualEnvRoot(sdkHome); - if (venv_root != null && venv_root.isDirectory()) { - File lib_root = new File(venv_root, "lib"); - if (lib_root.isDirectory()) { - String[] inside = lib_root.list(); - for (String s : inside) { - if (PYTHON_NN_RE.matcher(s).matches()) { - File py_lib_root = new File(lib_root, s); - if (new File(py_lib_root, "no-global-site-packages.txt").exists()) return; // don't add hardcoded paths - } - } - } - } - addHardcodedPaths(sdkUpdater); - } - } - - public static void updateUserAddedPaths(PySdkUpdater sdkUpdater) - throws InvalidSdkException { - SdkAdditionalData data = sdkUpdater.getSdk().getSdkAdditionalData(); - if (data instanceof PythonSdkAdditionalData) { - for (VirtualFile file : ((PythonSdkAdditionalData)data).getAddedPathFiles()) { - addSdkRoot(sdkUpdater, file); - } - } - } - - private static void addSkeletonsRoot(@NotNull PySdkUpdater sdkUpdater, String sdkHome) { - @NonNls final String skeletonsPath = getSkeletonsPath(PathManager.getSystemPath(), sdkHome); - new File(skeletonsPath).mkdirs(); - final VirtualFile builtins_root = LocalFileSystem.getInstance().refreshAndFindFileByPath(skeletonsPath); - assert builtins_root != null : "Cannot find skeletons path " + skeletonsPath + " in VFS"; - sdkUpdater.addRoot(builtins_root, BUILTIN_ROOT_TYPE); - } - - protected static void addHardcodedPaths(PySdkUpdater sdkUpdater) { - // Add python-django installed as package in Linux - // NOTE: fragile and arbitrary - if (SystemInfo.isLinux) { - final VirtualFile file = LocalFileSystem.getInstance().findFileByPath("/usr/lib/python-django"); - if (file != null) { - sdkUpdater.addRoot(file, OrderRootType.CLASSES); - } - } - } - - public static void addSdkRoot(PySdkUpdater sdkUpdater, String path) { - final VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByPath(path); - if (file != null) { - addSdkRoot(sdkUpdater, file); - } - else { - LOG.info("Bogus sys.path entry " + path); - } - } - - private static void addSdkRoot(@NotNull PySdkUpdater sdkUpdater, final @NotNull VirtualFile child) { - // NOTE: Files marked as library sources are not considered part of project source. Since the directory of the project the - // user is working on is included in PYTHONPATH with many configurations (e.g. virtualenv), we must not mark SDK paths as - // library sources, only as classes. - sdkUpdater.addRoot(getSdkRootVirtualFile(child), OrderRootType.CLASSES); - } - @NotNull public static VirtualFile getSdkRootVirtualFile(@NotNull VirtualFile path) { String suffix = path.getExtension(); diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java index cdb534b3c53e..5d32d0d175bb 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java @@ -15,12 +15,14 @@ */ package com.jetbrains.python.sdk; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.intellij.openapi.application.Application; import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.application.ModalityState; import com.intellij.openapi.application.PathManager; import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.extensions.Extensions; import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleManager; import com.intellij.openapi.progress.ProgressIndicator; @@ -28,17 +30,20 @@ import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.progress.Task; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; -import com.intellij.openapi.projectRoots.SdkTypeId; +import com.intellij.openapi.projectRoots.SdkAdditionalData; +import com.intellij.openapi.projectRoots.SdkModificator; import com.intellij.openapi.roots.OrderRootType; import com.intellij.openapi.startup.StartupActivity; +import com.intellij.openapi.ui.Messages; +import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.io.FileUtilRt; -import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.StandardFileSystems; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.PathMappingSettings; -import com.intellij.util.ui.UIUtil; +import com.intellij.util.concurrency.BlockingSet; import com.jetbrains.python.PyBundle; import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil; +import com.jetbrains.python.psi.PyUtil; import com.jetbrains.python.remote.PyRemoteSdkAdditionalDataBase; import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher; import org.jetbrains.annotations.NotNull; @@ -50,72 +55,48 @@ import java.util.*; import java.util.List; /** - * A component that initiates a refresh of all project's Python SDKs. - * Delegates most of the work to PythonSdkType. - *
+ * Refreshes all project's Python SDKs. * + * @author vlan * @author yole */ public class PythonSdkUpdater implements StartupActivity { private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.sdk.PythonSdkUpdater"); + public static final int INITIAL_ACTIVITY_DELAY = 7000; - public static PythonSdkUpdater getInstance() { - final StartupActivity[] extensions = Extensions.getExtensions(StartupActivity.POST_STARTUP_ACTIVITY); - for (StartupActivity extension : extensions) { - if (extension instanceof PythonSdkUpdater) { - return (PythonSdkUpdater)extension; - } - } - throw new UnsupportedOperationException("could not find self"); - } + private static final Object ourLock = new Object(); + private static final Set ourScheduledToRefresh = Sets.newHashSet(); + private static final BlockingSet ourUnderRefresh = new BlockingSet(); + /** + * Refreshes the SDKs of the modules for the open project after some delay. + */ @Override public void runActivity(@NotNull final Project project) { final Application application = ApplicationManager.getApplication(); if (application.isUnitTestMode()) { return; } - - updateActiveSdks(project, 7000); - } - - public static void updateActiveSdks(@NotNull final Project project, final int delay) { - final Set sdksToUpdate = new HashSet(); - for (Module module : ModuleManager.getInstance(project).getModules()) { - final Sdk sdk = PythonSdkType.findPythonSdk(module); - if (sdk != null) { - final SdkTypeId sdkType = sdk.getSdkType(); - if (sdkType instanceof PythonSdkType) { - sdksToUpdate.add(sdk); - } - } - } - - // NOTE: everything is run later on the AWT thread - if (!sdksToUpdate.isEmpty()) { - updateSdks(project, delay, sdksToUpdate); - } - } - - private static void updateSdks(final Project project, final int delay, final Set sdksToUpdate) { - ApplicationManager.getApplication().executeOnPooledThread(new Runnable() { + application.executeOnPooledThread(new Runnable() { + @Override public void run() { - if (delay > 0) { - try { - Thread.sleep(delay); // wait until all short-term disk-hitting activity ceases - } - catch (InterruptedException ignore) { - } + try { + Thread.sleep(INITIAL_ACTIVITY_DELAY); } - // update skeletons - ApplicationManager.getApplication().invokeLater(new Runnable() { + catch (InterruptedException ignored) { + } + application.invokeLater(new Runnable() { @Override public void run() { - ProgressManager.getInstance().run(new Task.Backgroundable(project, PyBundle.message("sdk.gen.updating.skels"), false) { + ProgressManager.getInstance().run(new Task.Backgroundable(project, "Updating Python Paths", false) { @Override public void run(@NotNull ProgressIndicator indicator) { - for (final Sdk sdk : sdksToUpdate) { - updateSdk(sdk, project); + final Project project = getProject(); + if (project.isDisposed()) { + return; + } + for (final Sdk sdk : getPythonSdks(project)) { + update(sdk, null, project, null); } } }); @@ -125,180 +106,299 @@ public class PythonSdkUpdater implements StartupActivity { }); } - public static void updateSdk(final Sdk sdk, final Project project) { - try { - LOG.info("Performing background update of skeletons for SDK " + sdk.getHomePath()); - updateSdk(project, null, PySdkUpdater.fromSdkPath(sdk.getHomePath())); + /** + * Updates the paths of an SDK and regenerates its skeletons as a background task. + * + * May be invoked from any thread. May freeze the current thread while evaluating sys.path. + * + * For a local SDK it commits all the SDK paths and runs a background task for updating skeletons. For a remote SDK it runs a background + * task for updating skeletons that saves path mappings in the additional SDK data and then commits all the SDK paths. + * + * The commit of the changes in the SDK happens in the AWT thread while the current thread is waiting the result. + * + * @param sdkModificator if null then it tries to get an SDK modifier from the SDK table, falling back to the modifier of the SDK + * passed as an argument accessed from the AWT thread + * @return false if there was an immediate problem updating the SDK. Other problems are reported as log entries and balloons. + */ + public static boolean update(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable final Project project, + @Nullable final Component ownerComponent) { + final String homePath = sdk.getHomePath(); + synchronized (ourLock) { + ourScheduledToRefresh.add(homePath); } - catch (PySdkUpdater.PySdkNotFoundException e) { - LOG.info("Sdk " + sdk.getName() + " was removed during update process."); + if (!updateLocalSdkPaths(sdk, sdkModificator)) { + return false; } - catch (InvalidSdkException e) { - if (PythonSdkType.isVagrant(sdk) || PythonSdkType.isDocker(sdk)) { - PythonSdkType.notifyRemoteSdkSkeletonsFail(e, new Runnable() { + ApplicationManager.getApplication().invokeLater(new Runnable() { + @Override + public void run() { + synchronized (ourLock) { + if (!ourScheduledToRefresh.contains(homePath)) { + return; + } + ourScheduledToRefresh.remove(homePath); + } + ProgressManager.getInstance().run(new Task.Backgroundable(project, PyBundle.message("sdk.gen.updating.skeletons"), false) { @Override - public void run() { - updateSdk(sdk, project); + public void run(@NotNull ProgressIndicator indicator) { + final Project project = getProject(); + final Sdk sdk = PythonSdkType.findSdkByPath(homePath); + if (sdk != null) { + ourUnderRefresh.put(homePath); + try { + final String skeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), homePath); + try { + if (PythonSdkType.isRemote(sdk) && project == null && ownerComponent == null) { + LOG.error("For refreshing skeletons of remote SDK, either project or owner component must be specified"); + } + LOG.info("Performing background update of skeletons for SDK " + sdk.getHomePath()); + PySkeletonRefresher.refreshSkeletonsOfSdk(project, ownerComponent, skeletonsPath, sdk); + updateRemoteSdkPaths(sdk); + } + catch (InvalidSdkException e) { + if (PythonSdkType.isVagrant(sdk) || PythonSdkType.isDocker(sdk)) { + PythonSdkType.notifyRemoteSdkSkeletonsFail(e, new Runnable() { + @Override + public void run() { + final Sdk sdk = PythonSdkType.findSdkByPath(homePath); + if (sdk != null) { + update(sdk, null, project, ownerComponent); + } + } + }); + } + else if (!PythonSdkType.isInvalid(sdk)) { + LOG.error(e); + } + } + } + finally { + ourUnderRefresh.remove(homePath); + } + } } }); } - else if (!PythonSdkType.isInvalid(sdk)) { - LOG.error(e); - } + }, ModalityState.NON_MODAL); + return true; + } + + /** + * Updates the paths of an SDK and regenerates its skeletons as a background task. Shows an error message if the update fails. + * + * @see {@link #update(Sdk, SdkModificator, Project, Component)} + */ + public static void updateOrShowError(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator, @Nullable Project project, + @Nullable Component ownerComponent) { + final boolean success = update(sdk, sdkModificator, project, ownerComponent); + if (!success) { + final String homePath = sdk.getHomePath(); + final String sdkName = homePath != null ? homePath : sdk.getName(); + Messages.showErrorDialog(project, + PyBundle.message("MSG.cant.setup.sdk.$0", FileUtil.toSystemDependentName(sdkName)), + PyBundle.message("MSG.title.bad.sdk")); } } - public static void updateSdk(@Nullable Project project, @Nullable Component ownerComponent, @NotNull final PySdkUpdater sdkUpdater) - throws InvalidSdkException { - String skeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), sdkUpdater.getHomePath()); - - PySkeletonRefresher.refreshSkeletonsOfSdk(project, ownerComponent, skeletonsPath, sdkUpdater); // NOTE: whole thing would need a rename - if (!PySdkUtil.isRemote(sdkUpdater.getSdk())) { - updateSysPath(sdkUpdater); + /** + * Updates the paths of a local SDK. + * + * May be invoked from any thread. May freeze the current thread while evaluating sys.path. + */ + private static boolean updateLocalSdkPaths(@NotNull Sdk sdk, @Nullable SdkModificator sdkModificator) { + if (!PythonSdkType.isRemote(sdk)) { + final List localSdkPaths; + try { + localSdkPaths = getLocalSdkPaths(sdk); + } + catch (InvalidSdkException e) { + if (!PythonSdkType.isInvalid(sdk)) { + LOG.error(e); + } + return false; + } + commitSdkPathsIfChanged(sdk, sdkModificator, localSdkPaths); } - else { - PyRemoteSdkAdditionalDataBase remoteSdkData = (PyRemoteSdkAdditionalDataBase)sdkUpdater.getSdk().getSdkAdditionalData(); - assert remoteSdkData != null; + return true; + } + + /** + * Updates the paths of a remote SDK. + * + * Requires the skeletons refresh steps to be run before it in order to get remote paths mappings in the additional SDK data. + * + * You may invoke it from any thread. Blocks until the commit is done in the AWT thread. + */ + private static void updateRemoteSdkPaths(Sdk sdk) { + if (PythonSdkType.isRemote(sdk)) { + final List remoteSdkPaths = getRemoteSdkPaths(sdk); + commitSdkPathsIfChanged(sdk, null, remoteSdkPaths); + } + } + + /** + * Returns all the paths for a local SDK. + */ + @NotNull + private static List getLocalSdkPaths(@NotNull Sdk sdk) throws InvalidSdkException { + return ImmutableList.builder() + .addAll(evaluateSysPath(sdk)) + .addAll(getSkeletonsPaths(sdk)) + .addAll(getUserAddedPaths(sdk)) + .build(); + } + + /** + * Returns all the paths for a remote SDK. + * + * Requires the skeletons refresh steps to be run before it in order to get remote paths mappings in the additional SDK data. + */ + @NotNull + private static List getRemoteSdkPaths(@NotNull Sdk sdk) { + return ImmutableList.builder() + .addAll(getRemoteSdkMappedPaths(sdk)) + .addAll(getSkeletonsPaths(sdk)) + .addAll(getUserAddedPaths(sdk)) + .build(); + } + + /** + * Returns all the paths manually added to an SDK by the user. + */ + @NotNull + private static List getUserAddedPaths(@NotNull Sdk sdk) { + final SdkAdditionalData additionalData = sdk.getSdkAdditionalData(); + final PythonSdkAdditionalData pythonAdditionalData = PyUtil.as(additionalData, PythonSdkAdditionalData.class); + return pythonAdditionalData != null ? Lists.newArrayList(pythonAdditionalData.getAddedPathFiles()) : + Collections.emptyList(); + } + + /** + * Returns local paths for a remote SDK that have been mapped to remote paths during the skeleton refresh step. + * + * Returns all the existing paths except those manually excluded by the user. + */ + @NotNull + private static List getRemoteSdkMappedPaths(@NotNull Sdk sdk) { + final SdkAdditionalData additionalData = sdk.getSdkAdditionalData(); + if (additionalData instanceof PyRemoteSdkAdditionalDataBase) { + final PyRemoteSdkAdditionalDataBase remoteSdkData = (PyRemoteSdkAdditionalDataBase)additionalData; final List paths = Lists.newArrayList(); for (PathMappingSettings.PathMapping mapping : remoteSdkData.getPathMappings().getPathMappings()) { paths.add(mapping.getLocalRoot()); } + return filterRootPaths(sdk, paths); + } + return Collections.emptyList(); + } - UIUtil.invokeAndWaitIfNeeded(new Runnable() { - @Override - public void run() { - updateSdkPath(sdkUpdater, paths); + /** + * Filters valid paths from an initial set of Python paths and returns them as virtual files. + */ + @NotNull + private static List filterRootPaths(@NotNull Sdk sdk, @NotNull List paths) { + final PythonSdkAdditionalData pythonAdditionalData = PyUtil.as(sdk.getSdkAdditionalData(), PythonSdkAdditionalData.class); + final Collection excludedPaths = pythonAdditionalData != null ? pythonAdditionalData.getExcludedPathFiles() : + Collections.emptyList(); + final List results = Lists.newArrayList(); + for (String path : paths) { + if (path != null && !FileUtilRt.extensionEquals(path, "egg-info")) { + final VirtualFile virtualFile = StandardFileSystems.local().refreshAndFindFileByPath(path); + if (virtualFile != null) { + final VirtualFile rootFile = PythonSdkType.getSdkRootVirtualFile(virtualFile); + if (!excludedPaths.contains(rootFile)) { + results.add(virtualFile); + continue; + } } - }); - } - } - - private static void updateSysPath(@NotNull final PySdkUpdater sdkUpdater) throws InvalidSdkException { - long start_time = System.currentTimeMillis(); - final List sysPath = PythonSdkType.getSysPath(sdkUpdater.getHomePath()); - final VirtualFile file = PyUserSkeletonsUtil.getUserSkeletonsDirectory(); - if (file != null) { - sysPath.add(file.getPath()); - } - UIUtil.invokeAndWaitIfNeeded(new Runnable() { - @Override - public void run() { - updateSdkPath(sdkUpdater, sysPath); - } - }); - LOG.info("Updating sys.path took " + (System.currentTimeMillis() - start_time) + " ms"); - } - - /** - * Updates SDK based on sys.path and cleans legacy information up. - */ - private static void updateSdkPath(@NotNull PySdkUpdater sdkUpdater, @NotNull List sysPath) { - addNewSysPathEntries(sdkUpdater, sysPath); - removeSourceRoots(sdkUpdater); - removeDuplicateClassRoots(sdkUpdater); - updateBinarySkeletonsPath(sdkUpdater); - updateUserSkeletonsPath(sdkUpdater); - } - - /** - * Adds new CLASSES entries found in sys.path. - */ - private static boolean addNewSysPathEntries(@NotNull PySdkUpdater sdkUpdater, @NotNull List sysPath) { - final List oldRoots = Arrays.asList(sdkUpdater.getSdk().getRootProvider().getFiles(OrderRootType.CLASSES)); - PythonSdkAdditionalData additionalData = sdkUpdater.getSdk().getSdkAdditionalData() instanceof PythonSdkAdditionalData - ? (PythonSdkAdditionalData)sdkUpdater.getSdk().getSdkAdditionalData() - : null; - List newRoots = new ArrayList(); - for (String root : sysPath) { - if (new File(root).exists() && - !FileUtilRt.extensionEquals(root, "egg-info") && - (additionalData == null || !wasOldRoot(root, additionalData.getExcludedPathFiles())) && - !wasOldRoot(root, oldRoots)) { - newRoots.add(root); } + LOG.info("Bogus sys.path entry " + path); } - if (!newRoots.isEmpty()) { - for (String root : newRoots) { - PythonSdkType.addSdkRoot(sdkUpdater, root); - } - return true; - } - return false; + return results; } /** - * Removes duplicate roots that have been added as the result of a bug with *.egg handling. + * Returns the paths of the binary skeletons and user skeletons for an SDK. */ - private static boolean removeDuplicateClassRoots(@NotNull PySdkUpdater sdkUpdater) { - final List sourceRoots = Arrays.asList(sdkUpdater.getRoots(OrderRootType.CLASSES)); - final LinkedHashSet uniqueRoots = new LinkedHashSet(sourceRoots); - if (uniqueRoots.size() != sourceRoots.size()) { - sdkUpdater.removeRoots(OrderRootType.CLASSES); - for (VirtualFile root : uniqueRoots) { - sdkUpdater.addRoot(root, OrderRootType.CLASSES); - } - return true; - } - return false; - } - - /** - * Removes legacy SOURCES entries in Python SDK tables (PY-2891). - */ - private static boolean removeSourceRoots(@NotNull PySdkUpdater sdkUpdater) { - final VirtualFile[] sourceRoots = sdkUpdater.getRoots(OrderRootType.SOURCES); - if (sourceRoots.length > 0) { - sdkUpdater.removeRoots(OrderRootType.SOURCES); - return true; - } - return false; - } - - /** - * Updates user skeletons path in the Python SDK table. - */ - private static void updateUserSkeletonsPath(@NotNull PySdkUpdater sdkUpdater) { - updateSkeletonsPath(sdkUpdater, PyUserSkeletonsUtil.getUserSkeletonsDirectory(), PyUserSkeletonsUtil.USER_SKELETONS_DIR, - "User skeletons"); - } - - /** - * Updates binary skeletons path in the Python SDK table. - */ - private static void updateBinarySkeletonsPath(@NotNull PySdkUpdater sdkUpdater) { - final String skeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), sdkUpdater.getHomePath()); + @NotNull + private static List getSkeletonsPaths(@NotNull Sdk sdk) { + final List results = Lists.newArrayList(); + final String skeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), sdk.getHomePath()); if (skeletonsPath != null) { + //noinspection ResultOfMethodCallIgnored + new File(skeletonsPath).mkdirs(); final VirtualFile skeletonsDir = StandardFileSystems.local().refreshAndFindFileByPath(skeletonsPath); if (skeletonsDir != null) { - updateSkeletonsPath(sdkUpdater, skeletonsDir, PythonSdkType.SKELETON_DIR_NAME, "Binary skeletons"); + results.add(skeletonsDir); + LOG.info("Binary skeletons directory for SDK \"" + sdk.getName() + "\" (" + sdk.getHomePath() + "): " + + skeletonsDir.getPath()); } } + final VirtualFile userSkeletonsDir = PyUserSkeletonsUtil.getUserSkeletonsDirectory(); + if (userSkeletonsDir != null) { + results.add(userSkeletonsDir); + LOG.info("User skeletons directory for SDK \"" + sdk.getName() + "\" (" + sdk.getHomePath() + "): " + + userSkeletonsDir.getPath()); + } + return results; } - private static void updateSkeletonsPath(@NotNull PySdkUpdater sdkUpdater, - @Nullable VirtualFile skeletonsDir, - @NotNull String skeletonsDirPattern, - @NotNull String skeletonsTitle) { - if (skeletonsDir != null) { - LOG.info(skeletonsTitle + " directory for SDK \"" + sdkUpdater.getSdk().getName() + "\" (" + sdkUpdater.getHomePath() + "): " + - skeletonsDir.getPath()); - final List sourceRoots = Arrays.asList(sdkUpdater.getRoots(OrderRootType.CLASSES)); - sdkUpdater.removeRoots(OrderRootType.CLASSES); - for (final VirtualFile root : sourceRoots) { - if (!root.getPath().contains(skeletonsDirPattern)) { - sdkUpdater.addRoot(root, OrderRootType.CLASSES); + /** + * Evaluates sys.path by running the Python interpreter from a local SDK. + * + * Returns all the existing paths except those manually excluded by the user. + */ + @NotNull + private static List evaluateSysPath(@NotNull Sdk sdk) throws InvalidSdkException { + if (PythonSdkType.isRemote(sdk)) { + throw new IllegalArgumentException("Cannot evaluate sys.path for remote Python interpreter " + sdk); + } + final long startTime = System.currentTimeMillis(); + final List sysPath = PythonSdkType.getSysPath(sdk.getHomePath()); + LOG.info("Updating sys.path took " + (System.currentTimeMillis() - startTime) + " ms"); + return filterRootPaths(sdk, sysPath); + } + + /** + * Commits new SDK paths using an SDK modificator if the paths have been changed. + * + * You may invoke it from any thread. Blocks until the commit is done in the AWT thread. + */ + private static void commitSdkPathsIfChanged(@NotNull Sdk sdk, + @Nullable final SdkModificator sdkModificator, + @NotNull final List sdkPaths) { + final String homePath = sdk.getHomePath(); + final SdkModificator modificatorToGetRoots = sdkModificator != null ? sdkModificator : sdk.getSdkModificator(); + final List currentSdkPaths = Arrays.asList(modificatorToGetRoots.getRoots(OrderRootType.CLASSES)); + if (!Sets.difference(Sets.newHashSet(sdkPaths), Sets.newHashSet(currentSdkPaths)).isEmpty()) { + ApplicationManager.getApplication().invokeAndWait(new Runnable() { + @Override + public void run() { + final Sdk sdk = PythonSdkType.findSdkByPath(homePath); + final SdkModificator modificatorToCommit = sdkModificator != null ? sdkModificator : + sdk != null ? sdk.getSdkModificator() : modificatorToGetRoots; + modificatorToCommit.removeAllRoots(); + for (VirtualFile sdkPath : sdkPaths) { + modificatorToCommit.addRoot(PythonSdkType.getSdkRootVirtualFile(sdkPath), OrderRootType.CLASSES); + } + modificatorToCommit.commitChanges(); } - } - sdkUpdater.addRoot(skeletonsDir, OrderRootType.CLASSES); + }, ModalityState.defaultModalityState()); } } - private static boolean wasOldRoot(@NotNull String root, @NotNull Collection oldRoots) { - final VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByPath(root); - if (file != null) { - final VirtualFile rootFile = PythonSdkType.getSdkRootVirtualFile(file); - return oldRoots.contains(rootFile); + /** + * Returns unique Python SDKs for the open modules of the project. + */ + @NotNull + private static Set getPythonSdks(@NotNull Project project) { + final Set pythonSdks = Sets.newLinkedHashSet(); + for (Module module : ModuleManager.getInstance(project).getModules()) { + final Sdk sdk = PythonSdkType.findPythonSdk(module); + if (sdk != null && sdk.getSdkType() instanceof PythonSdkType) { + pythonSdks.add(sdk); + } } - return false; + return pythonSdks; } } diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java index 6549a3f49114..c9c7d636a1a1 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java @@ -47,7 +47,6 @@ import com.jetbrains.python.packaging.PyPackageManager; import com.jetbrains.python.psi.resolve.PythonSdkPathCache; import com.jetbrains.python.remote.PythonRemoteInterpreterManager; import com.jetbrains.python.sdk.InvalidSdkException; -import com.jetbrains.python.sdk.PySdkUpdater; import com.jetbrains.python.sdk.PySdkUtil; import com.jetbrains.python.sdk.PythonSdkType; import org.jetbrains.annotations.NonNls; @@ -112,25 +111,25 @@ public class PySkeletonRefresher { public static void refreshSkeletonsOfSdk(@Nullable Project project, Component ownerComponent, String skeletonsPath, - @NotNull PySdkUpdater sdkUpdater) + @NotNull Sdk sdk) throws InvalidSdkException { final Map> errors = new TreeMap>(); final List failedSdks = new SmartList(); final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator(); - final String homePath = sdkUpdater.getHomePath(); + final String homePath = sdk.getHomePath(); if (skeletonsPath == null) { LOG.info("Could not find skeletons path for SDK path " + homePath); } else { LOG.info("Refreshing skeletons for " + homePath); SkeletonVersionChecker checker = new SkeletonVersionChecker(0); // this default version won't be used - final PySkeletonRefresher refresher = new PySkeletonRefresher(project, ownerComponent, sdkUpdater.getSdk(), skeletonsPath, indicator, null); + final PySkeletonRefresher refresher = new PySkeletonRefresher(project, ownerComponent, sdk, skeletonsPath, indicator, null); changeGeneratingSkeletons(1); try { List sdkErrors = refresher.regenerateSkeletons(checker); if (sdkErrors.size() > 0) { - String sdkName = sdkUpdater.getSdk().getName(); + String sdkName = sdk.getName(); List knownErrors = errors.get(sdkName); if (knownErrors == null) { errors.put(sdkName, sdkErrors); diff --git a/python/testSrc/com/jetbrains/python/PythonMockSdk.java b/python/testSrc/com/jetbrains/python/PythonMockSdk.java index fb4541deea90..423d1e67c20f 100644 --- a/python/testSrc/com/jetbrains/python/PythonMockSdk.java +++ b/python/testSrc/com/jetbrains/python/PythonMockSdk.java @@ -27,7 +27,6 @@ import com.intellij.psi.stubs.StubUpdatingIndex; import com.intellij.util.indexing.FileBasedIndex; import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil; import com.jetbrains.python.psi.stubs.PyModuleNameIndex; -import com.jetbrains.python.sdk.PySdkUpdater; import com.jetbrains.python.sdk.PythonSdkType; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; @@ -75,7 +74,7 @@ public class PythonMockSdk { sdkModificator.addRoot(LocalFileSystem.getInstance().refreshAndFindFileByIoFile(libPath), OrderRootType.CLASSES); } - PyUserSkeletonsUtil.addUserSkeletonsRoot(PySdkUpdater.fromSdkModificator(sdk, sdkModificator)); + sdkModificator.addRoot(PyUserSkeletonsUtil.getUserSkeletonsDirectory(), OrderRootType.CLASSES); String mock_stubs_path = mock_path + PythonSdkType.SKELETON_DIR_NAME; sdkModificator.addRoot(LocalFileSystem.getInstance().refreshAndFindFileByPath(mock_stubs_path), PythonSdkType.BUILTIN_ROOT_TYPE); diff --git a/python/testSrc/com/jetbrains/python/sdkTools/PyTestSdkTools.java b/python/testSrc/com/jetbrains/python/sdkTools/PyTestSdkTools.java index 35dc7b2e3d49..ab7027f833a4 100644 --- a/python/testSrc/com/jetbrains/python/sdkTools/PyTestSdkTools.java +++ b/python/testSrc/com/jetbrains/python/sdkTools/PyTestSdkTools.java @@ -11,10 +11,10 @@ import com.intellij.openapi.roots.OrderRootType; import com.intellij.openapi.roots.ProjectRootManager; import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.testFramework.UsefulTestCase; import com.jetbrains.python.sdk.InvalidSdkException; -import com.jetbrains.python.sdk.PySdkUpdater; import com.jetbrains.python.sdk.PythonSdkType; import com.jetbrains.python.sdk.skeletons.PySkeletonRefresher; import com.jetbrains.python.sdk.skeletons.SkeletonVersionChecker; @@ -115,7 +115,7 @@ public final class PyTestSdkTools { modificator.removeRoots(OrderRootType.CLASSES); for (final String path : PythonSdkType.getSysPathsFromScript(sdk.getHomePath())) { - PythonSdkType.addSdkRoot(PySdkUpdater.fromSdkModificator(sdk, modificator), path); + addTestSdkRoot(modificator, path); } if (!addSkeletons) { UsefulTestCase.edt(new Runnable() { @@ -131,7 +131,7 @@ public final class PyTestSdkTools { final File skeletonsDir = new File(tempDir, PythonSdkType.SKELETON_DIR_NAME); FileUtil.createDirectory(skeletonsDir); final String skeletonsPath = skeletonsDir.toString(); - PythonSdkType.addSdkRoot(PySdkUpdater.fromSdkModificator(sdk, modificator), skeletonsPath); + addTestSdkRoot(modificator, skeletonsPath); UsefulTestCase.edt(new Runnable() { @Override @@ -146,4 +146,11 @@ public final class PyTestSdkTools { final List errors = refresher.regenerateSkeletons(checker); Assert.assertThat("Errors found", errors, Matchers.empty()); } + + public static void addTestSdkRoot(@NotNull SdkModificator sdkModificator, @NotNull String path) { + final VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByPath(path); + if (file != null) { + sdkModificator.addRoot(PythonSdkType.getSdkRootVirtualFile(file), OrderRootType.CLASSES); + } + } }