diff --git a/python/python-pyproject/BUILD.bazel b/python/python-pyproject/BUILD.bazel index f9012b208f4c..5afe25bdd591 100644 --- a/python/python-pyproject/BUILD.bazel +++ b/python/python-pyproject/BUILD.bazel @@ -85,6 +85,8 @@ jvm_library( "//platform/testFramework/junit5/projectStructure", "//platform/testFramework/junit5/projectStructure:projectStructure_test_lib", "//python/junit5Tests-framework:community-junit5Tests-framework_test_lib", + "//platform/testFramework", + "//platform/testFramework:testFramework_test_lib", ], exports = ["@lib//:tuweni-toml"] ) diff --git a/python/python-pyproject/intellij.python.pyproject.iml b/python/python-pyproject/intellij.python.pyproject.iml index ba3b26270448..9bd45a1bb3e7 100644 --- a/python/python-pyproject/intellij.python.pyproject.iml +++ b/python/python-pyproject/intellij.python.pyproject.iml @@ -38,5 +38,6 @@ + \ No newline at end of file diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/FsInfoStorage.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/FsInfoStorage.kt deleted file mode 100644 index 597709bca805..000000000000 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/FsInfoStorage.kt +++ /dev/null @@ -1,26 +0,0 @@ -package com.intellij.python.pyproject.model.internal.autoImportBridge - -import com.intellij.python.pyproject.model.internal.pyProjectToml.FSWalkInfo -import com.intellij.python.pyproject.model.internal.pyProjectToml.walkFileSystem -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import java.nio.file.Path - -internal class FsInfoStorage(private val projectRootDir: Path) { - private val updateMut = Mutex() - - private var cache: FSWalkInfo? = null - - - suspend fun getFsInfo(forceRefresh: Boolean): FSWalkInfo { - updateMut.withLock { - val cache = cache - if (cache != null && !forceRefresh) { - return cache - } - val fsInfo = walkFileSystem(projectRootDir) - this.cache = fsInfo - return fsInfo - } - } -} \ No newline at end of file diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyExternalSystemProjectAware.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyExternalSystemProjectAware.kt index c27953b8c739..92632119534c 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyExternalSystemProjectAware.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyExternalSystemProjectAware.kt @@ -1,19 +1,25 @@ package com.intellij.python.pyproject.model.internal.autoImportBridge import com.intellij.openapi.Disposable +import com.intellij.openapi.application.writeAction +import com.intellij.openapi.components.Service import com.intellij.openapi.components.service import com.intellij.openapi.externalSystem.autoimport.* import com.intellij.openapi.externalSystem.model.ProjectSystemId import com.intellij.openapi.progress.runBlockingMaybeCancellable import com.intellij.openapi.project.Project -import com.intellij.openapi.project.guessProjectDir import com.intellij.platform.backend.observation.launchTracked +import com.intellij.project.stateStore +import com.intellij.psi.PsiDocumentManager import com.intellij.python.pyproject.model.api.ModelRebuiltListener import com.intellij.python.pyproject.model.internal.PyProjectTomlBundle +import com.intellij.python.pyproject.model.internal.pyProjectToml.walkFileSystemNoTomlContent +import com.intellij.python.pyproject.model.internal.pyProjectToml.walkFileSystemWithTomlContent import com.intellij.python.pyproject.model.internal.workspaceBridge.rebuildProjectModel import com.intellij.util.concurrency.annotations.RequiresBackgroundThread import com.intellij.util.messages.Topic import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.nio.file.Path @@ -21,17 +27,17 @@ import kotlin.io.path.pathString internal class PyExternalSystemProjectAware private constructor( private val project: Project, - projectRootDir: Path, + private val projectRootDir: Path, ) : ExternalSystemProjectAware { override val projectId: ExternalSystemProjectId = ExternalSystemProjectId(SYSTEM_ID, projectRootDir.pathString) - private val fsInfo = FsInfoStorage(projectRootDir) @get:RequiresBackgroundThread override val settingsFiles: Set get() = runBlockingMaybeCancellable { - val fsInfo = fsInfo.getFsInfo(forceRefresh = true) - return@runBlockingMaybeCancellable fsInfo.tomlFiles.keys.map { it.pathString }.toSet() + // We do not need file content: only names here. + val fsInfo = walkFileSystemNoTomlContent(projectRootDir) + return@runBlockingMaybeCancellable fsInfo.rawTomlFiles.map { it.pathString }.toSet() } override fun subscribe(listener: ExternalSystemProjectListener, parentDisposable: Disposable) { @@ -39,10 +45,18 @@ internal class PyExternalSystemProjectAware private constructor( } override fun reloadProject(context: ExternalSystemProjectReloadContext) { - project.service().scope.launchTracked { + reloadProjectImpl() + } + + internal fun reloadProjectImpl() { + project.service().scope.launchTracked { + writeAction { + // We might get stale files otherwise + PsiDocumentManager.getInstance(project).commitAllDocuments() + } project.messageBus.syncAndPreloadPublisher(PROJECT_AWARE_TOPIC).apply { try { - val files = fsInfo.getFsInfo(forceRefresh = context.hasUndefinedModifications) + val files = walkFileSystemWithTomlContent(projectRootDir) this.onProjectReloadStart() rebuildProjectModel(project, files) this.onProjectReloadFinish(ExternalSystemRefreshStatus.SUCCESS) @@ -65,9 +79,14 @@ internal class PyExternalSystemProjectAware private constructor( companion object { + /** + * [project] can't be default, be sure to check it + */ internal suspend fun create(project: Project): PyExternalSystemProjectAware { + assert(!project.isDefault) { "Default project not supported" } val baseDir = withContext(Dispatchers.IO) { - project.guessProjectDir()?.toNioPath() ?: error("Project $project has no base dir") + // guessPath doesn't work: it returns first module path + project.stateStore.projectBasePath } return PyExternalSystemProjectAware(project, baseDir) } @@ -83,4 +102,7 @@ private val PROJECT_AWARE_TOPIC: Topic = Topic(ExternalSystemProjectListener::class.java, Topic.BroadcastDirection.NONE) @Topic.ProjectLevel -private val MODEL_REBUILD: Topic = Topic(ModelRebuiltListener::class.java, Topic.BroadcastDirection.NONE) \ No newline at end of file +internal val MODEL_REBUILD: Topic = Topic(ModelRebuiltListener::class.java, Topic.BroadcastDirection.NONE) + +@Service(Service.Level.PROJECT) +private class PyExternalSystemProjectAwareService(val scope: CoroutineScope) \ No newline at end of file diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyProjectAutoImportService.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyProjectAutoImportService.kt index 5337e0936555..79d7dca752d2 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyProjectAutoImportService.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/autoImportBridge/PyProjectAutoImportService.kt @@ -5,10 +5,16 @@ import com.intellij.openapi.components.Service import com.intellij.openapi.externalSystem.autoimport.ExternalSystemProjectId import com.intellij.openapi.externalSystem.autoimport.ExternalSystemProjectTracker import com.intellij.openapi.project.Project -import kotlinx.coroutines.CoroutineScope +/** + * [project] can't be default, check for it + */ @Service(Service.Level.PROJECT) -internal class PyProjectAutoImportService(private val project: Project, internal val scope: CoroutineScope) : Disposable { +internal class PyProjectAutoImportService(private val project: Project) : Disposable { + init { + assert(!project.isDefault) { "Default project not supported" } + } + @Volatile private lateinit var projectId: ExternalSystemProjectId diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectSyncActivity.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectSyncActivity.kt index 27c15f54f8f9..778a594eec55 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectSyncActivity.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectSyncActivity.kt @@ -15,6 +15,7 @@ internal class PyProjectSyncActivity : ProjectActivity { } override suspend fun execute(project: Project) { + if (project.isDefault) return // Service doesn't support default project project.service().start() } } \ No newline at end of file diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectTomlSyncAction.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectTomlSyncAction.kt index d937a4ab5b6a..99efdd6cdc8d 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectTomlSyncAction.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/platformBridge/PyProjectTomlSyncAction.kt @@ -4,7 +4,6 @@ import com.intellij.openapi.actionSystem.ActionUpdateThread import com.intellij.openapi.actionSystem.AnAction import com.intellij.openapi.actionSystem.AnActionEvent import com.intellij.openapi.components.service -import com.intellij.openapi.externalSystem.autoimport.ExternalSystemProjectTracker import com.intellij.python.pyproject.model.internal.autoImportBridge.PyProjectAutoImportService import com.intellij.python.pyproject.model.internal.projectModelEnabled @@ -12,14 +11,15 @@ internal class PyProjectTomlSyncAction : AnAction() { override fun actionPerformed(e: AnActionEvent) { val project = e.project ?: return - if (!projectModelEnabled) { + if (!projectModelEnabled || project.isDefault) { // Service doesn't support default project return } project.service().refresh() } override fun update(e: AnActionEvent) { - e.presentation.isEnabledAndVisible = e.project != null && projectModelEnabled + val project = e.project + e.presentation.isEnabledAndVisible = project != null && projectModelEnabled && !project.isDefault } override fun getActionUpdateThread(): ActionUpdateThread = ActionUpdateThread.BGT diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/FSWalkInfo.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/FSWalkInfo.kt deleted file mode 100644 index bcde354a1869..000000000000 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/FSWalkInfo.kt +++ /dev/null @@ -1,7 +0,0 @@ -package com.intellij.python.pyproject.model.internal.pyProjectToml - -import com.intellij.python.pyproject.PyProjectToml -import com.jetbrains.python.venvReader.Directory -import java.nio.file.Path - -internal data class FSWalkInfo(val tomlFiles: Map, val excludeDir: Set) \ No newline at end of file diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/FSWalkInfoWithToml.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/FSWalkInfoWithToml.kt new file mode 100644 index 000000000000..4e2e0d6485f7 --- /dev/null +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/FSWalkInfoWithToml.kt @@ -0,0 +1,12 @@ +package com.intellij.python.pyproject.model.internal.pyProjectToml + +import com.intellij.python.pyproject.PyProjectToml +import com.jetbrains.python.venvReader.Directory +import java.nio.file.Path + +// Files with toml content +internal data class FSWalkInfoWithToml(val tomlFiles: Map, val excludeDir: Set) + + +// Files only +internal data class FsWalkInfoNoToml(val rawTomlFiles: List, val excludedDirs: List) diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/tomFileTools.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/tomFileTools.kt index 768ece78f514..d77d170ff494 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/tomFileTools.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/pyProjectToml/tomFileTools.kt @@ -1,6 +1,7 @@ package com.intellij.python.pyproject.model.internal.pyProjectToml import com.intellij.openapi.diagnostic.fileLogger +import com.intellij.psi.PsiDocumentManager import com.intellij.python.pyproject.PY_PROJECT_TOML import com.intellij.python.pyproject.PyProjectToml import com.intellij.python.pyproject.model.spi.ProjectName @@ -24,20 +25,33 @@ import kotlin.io.path.visitFileTree // Tools to walk FS and parse pyproject.toml -internal suspend fun walkFileSystem(root: Directory): FSWalkInfo { - val files = ArrayList(10) - val excludeDir = ArrayList(10) +internal suspend fun walkFileSystemWithTomlContent(root: Directory): FSWalkInfoWithToml { + val (rawTomlFiles, excludedDirs) = walkFileSystemNoTomlContent(root) + + // TODO: with a big number of files, use `chunk` to parse them concurrently + val tomlFiles = rawTomlFiles.map { file -> + val toml = readFile(file) ?: return@map null + file to toml + }.filterNotNull().toMap() + return FSWalkInfoWithToml(tomlFiles = tomlFiles, excludedDirs.toSet()) +} + +internal suspend fun walkFileSystemNoTomlContent( + root: Directory, +): FsWalkInfoNoToml { + val excludedDirs = ArrayList(10) + val rawTomlFiles = ArrayList(10) withContext(Dispatchers.IO) { root.visitFileTree { onVisitFile { file, _ -> if (file.name == PY_PROJECT_TOML) { - files.add(file) + rawTomlFiles.add(file) } return@onVisitFile FileVisitResult.CONTINUE } onPostVisitDirectory { directory, _ -> return@onPostVisitDirectory if (directory.name.startsWith(".")) { - excludeDir.add(directory) + excludedDirs.add(directory) FileVisitResult.SKIP_SUBTREE } else { @@ -46,15 +60,11 @@ internal suspend fun walkFileSystem(root: Directory): FSWalkInfo { } } } - // TODO: with a big number of files, use `chunk` to parse them concurrently - val tomlFiles = files.map { file -> - val toml = readFile(file) ?: return@map null - file to toml - }.filterNotNull().toMap() - return FSWalkInfo(tomlFiles = tomlFiles, excludeDir.toSet()) + return FsWalkInfoNoToml(rawTomlFiles = rawTomlFiles, excludedDirs = excludedDirs) } - suspend fun getProjectStructureDefault( + +suspend fun getProjectStructureDefault( entries: Map, rootIndex: Map, ): ProjectStructureInfo = withContext(Dispatchers.Default) { diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceBridge/workspaceTools.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceBridge/workspaceTools.kt index b569ac4dd9df..4fe5dbbbc859 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceBridge/workspaceTools.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceBridge/workspaceTools.kt @@ -13,7 +13,7 @@ import com.intellij.platform.workspace.storage.url.VirtualFileUrlManager import com.intellij.python.common.tools.ToolId import com.intellij.python.pyproject.PyProjectToml import com.intellij.python.pyproject.model.internal.PyProjectTomlBundle -import com.intellij.python.pyproject.model.internal.pyProjectToml.FSWalkInfo +import com.intellij.python.pyproject.model.internal.pyProjectToml.FSWalkInfoWithToml import com.intellij.python.pyproject.model.internal.pyProjectToml.getProjectStructureDefault import com.intellij.python.pyproject.model.spi.ProjectName import com.intellij.python.pyproject.model.spi.PyProjectTomlProject @@ -21,6 +21,7 @@ import com.intellij.python.pyproject.model.spi.Tool import com.intellij.python.pyproject.model.spi.WorkspaceName import com.intellij.workspaceModel.ide.impl.legacyBridge.facet.FacetModelBridge.Companion.findFacet import com.intellij.workspaceModel.ide.impl.legacyBridge.sdk.SdkBridgeImpl.Companion.findSdkEntity +import com.intellij.workspaceModel.ide.isEqualOrParentOf import com.jetbrains.python.PyNames import com.jetbrains.python.facet.PythonFacetSettings import com.jetbrains.python.venvReader.Directory @@ -36,19 +37,41 @@ import kotlin.io.path.name // Workspace adapter functions -internal suspend fun rebuildProjectModel(project: Project, files: FSWalkInfo) { +internal suspend fun rebuildProjectModel(project: Project, files: FSWalkInfoWithToml) { changeWorkspaceMutex.withLock { - val entries = generatePyProjectTomlEntries(files) + val (entries, excludeDirs) = generatePyProjectTomlEntries(files) val newStorage = createEntityStorage(entries, project.workspaceModel.getVirtualFileUrlManager()) - project.workspaceModel.update(PyProjectTomlBundle.message("action.PyProjectTomlSyncAction.description")) { currentStorage -> // Fake module entity is added by default if nothing was discovered + val workspaceModel = project.workspaceModel + workspaceModel.update(PyProjectTomlBundle.message("action.PyProjectTomlSyncAction.description")) { currentStorage -> // Fake module entity is added by default if nothing was discovered removeFakeModuleEntity(currentStorage, entries.map { it.name.name }.toSet()) + // TODO: Store old module->SDK, so we can restoe it even when modules are destroyed relocateUserDefinedModuleSdk(currentStorage) { currentStorage.replaceBySource({ it is PyProjectTomlEntitySource }, newStorage) + + // Exclude dirs + if (excludeDirs.isEmpty()) return@relocateUserDefinedModuleSdk + val modules = currentStorage.entities(ModuleEntity::class.java).toList() + for (excludedRoot in excludeDirs.map { it.toVirtualFileUrl(workspaceModel.getVirtualFileUrlManager()) }) { + currentStorage.excludeRoot(excludedRoot, modules) + } } } } +} +private fun MutableEntityStorage.excludeRoot(rootToExclude: VirtualFileUrl, modules: List) { + for (moduleEntry in modules) { + for (rootEntity in moduleEntry.contentRoots) { + if (rootEntity.url.isEqualOrParentOf(rootToExclude) && rootToExclude !in rootEntity.excludedUrls.map { it.url }) { + modifyContentRootEntity(rootEntity) { + excludedUrls = excludedUrls + listOf( + ExcludeUrlEntity(rootToExclude, entitySource)) + } + return + } + } + } } /** @@ -89,9 +112,10 @@ internal fun relocateUserDefinedModuleSdk(storage: MutableEntityStorage, transfe } } +// TODO: DOC private suspend fun generatePyProjectTomlEntries( - fsInfo: FSWalkInfo, -): Set = withContext(Dispatchers.Default) { + fsInfo: FSWalkInfoWithToml, +): Pair, Set> = withContext(Dispatchers.Default) { val (files, allExcludeDirs) = fsInfo val entries = ArrayList() val usedNamed = mutableSetOf() @@ -148,7 +172,7 @@ private suspend fun generatePyProjectTomlEntries( entriesByName[member]!!.relationsWithTools.add(PyProjectTomlToolRelation.WorkspaceMember(tool.id, workspace)) } } - return@withContext entries.toSet() + return@withContext Pair(entries.toSet(), allExcludeDirs) } private suspend fun getNameFromEP(projectToml: PyProjectToml): Pair? = withContext(Dispatchers.Default) { diff --git a/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PyExternalSystemProjectAwareTest.kt b/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PyExternalSystemProjectAwareTest.kt new file mode 100644 index 000000000000..c5c071d7d6ea --- /dev/null +++ b/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PyExternalSystemProjectAwareTest.kt @@ -0,0 +1,70 @@ +package com.intellij.python.junit5Tests.unit.pyproject + +import com.intellij.openapi.Disposable +import com.intellij.openapi.project.modules +import com.intellij.python.pyproject.PY_PROJECT_TOML +import com.intellij.python.pyproject.model.api.ModelRebuiltListener +import com.intellij.python.pyproject.model.internal.autoImportBridge.MODEL_REBUILD +import com.intellij.python.pyproject.model.internal.autoImportBridge.PyExternalSystemProjectAware +import com.intellij.testFramework.common.timeoutRunBlocking +import com.intellij.testFramework.junit5.TestApplication +import com.intellij.testFramework.junit5.TestDisposable +import com.intellij.testFramework.junit5.fixture.projectFixture +import com.intellij.testFramework.junit5.fixture.tempPathFixture +import com.intellij.util.io.write +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.withContext +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Timeout +import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.minutes + + +@Timeout(TIMEOUT_MIN.toLong(), unit = TimeUnit.MINUTES) +@TestApplication +class PyExternalSystemProjectAwareTest { + private val members = arrayOf("foo", "bar").sortedArray() + + private val pathFixture = tempPathFixture() + private val projectFixture = projectFixture(pathFixture) + + @BeforeEach + fun prepareWorkspace(): Unit = timeoutRunBlocking { + val root = pathFixture.get() + for (member in members) { + root.resolve(member).resolve(PY_PROJECT_TOML).write(""" + [project] + name = "$member" + """.trimIndent()) + } + } + + + @Test + fun testBuildProject(@TestDisposable disposable: Disposable): Unit = timeoutRunBlocking(TIMEOUT_MIN.minutes) { + val sut = PyExternalSystemProjectAware.create(projectFixture.get()) + val files = withContext(Dispatchers.IO) { + sut.settingsFiles + } + Assertions.assertEquals(members.size, files.size, "Wrong number of toml files") + + sut.reloadProjectImpl() + + val m = Mutex(locked = true) + projectFixture.get().messageBus.connect(disposable).subscribe(MODEL_REBUILD, ModelRebuiltListener { project -> + try { + val moduleNames = project.modules.map { it.name }.sorted().toTypedArray() + Assertions.assertArrayEquals(members, moduleNames, "Wrong modules created") + } + finally { + m.unlock() + } + }) + m.lock() + } +} + +private const val TIMEOUT_MIN = 2 \ No newline at end of file diff --git a/python/python-pyproject/testResources/intellij.python.pyproject._test.xml b/python/python-pyproject/testResources/intellij.python.pyproject._test.xml index eb250dd9c851..442b0852f4f7 100644 --- a/python/python-pyproject/testResources/intellij.python.pyproject._test.xml +++ b/python/python-pyproject/testResources/intellij.python.pyproject._test.xml @@ -5,5 +5,6 @@ + \ No newline at end of file diff --git a/python/python-sdk-configurator/backend/resources/intellij.python.sdkConfigurator.backend.xml b/python/python-sdk-configurator/backend/resources/intellij.python.sdkConfigurator.backend.xml index 02c3050e282a..8f5a7b317f12 100644 --- a/python/python-sdk-configurator/backend/resources/intellij.python.sdkConfigurator.backend.xml +++ b/python/python-sdk-configurator/backend/resources/intellij.python.sdkConfigurator.backend.xml @@ -1,4 +1,4 @@ - + @@ -11,7 +11,7 @@ messages.PySdkConfiguratorBundle - diff --git a/python/python-sdk-configurator/backend/src/impl/ModuleConfigurationMode.kt b/python/python-sdk-configurator/backend/src/impl/ModuleConfigurationMode.kt new file mode 100644 index 000000000000..a8ca05992aec --- /dev/null +++ b/python/python-sdk-configurator/backend/src/impl/ModuleConfigurationMode.kt @@ -0,0 +1,20 @@ +package com.intellij.python.sdkConfigurator.backend.impl + +/** + * SDK for project modules can be configured in two different modes (see PY-86454) + */ +internal enum class ModuleConfigurationMode { + /** + * Do not bother user with questions, do not touch filesystem. + * For one-module project, configure SDK if files (i.e. venv) already exist. + * For multi-module only set SDK for workspace members if workspace parent SDK is set. + * + * In other words: No more than 1 SDK could be created. No files should be created. + */ + AUTOMATIC, + + /** + * Show user list of modules without SDK, ask them which to configure, and do so. + */ + INTERACTIVE +} diff --git a/python/python-sdk-configurator/backend/src/impl/ModulesSdkConfigurator.kt b/python/python-sdk-configurator/backend/src/impl/ModulesSdkConfigurator.kt index 0dd0dda43140..fc6b55343b59 100644 --- a/python/python-sdk-configurator/backend/src/impl/ModulesSdkConfigurator.kt +++ b/python/python-sdk-configurator/backend/src/impl/ModulesSdkConfigurator.kt @@ -2,6 +2,7 @@ package com.intellij.python.sdkConfigurator.backend.impl import com.intellij.openapi.diagnostic.debug import com.intellij.openapi.diagnostic.fileLogger +import com.intellij.openapi.diagnostic.trace import com.intellij.openapi.module.Module import com.intellij.openapi.project.Project import com.intellij.openapi.project.guessModuleDir @@ -31,9 +32,10 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.withContext +import org.jetbrains.annotations.ApiStatus /** - * Configures SDK for modules in [project]. + * Configures SDK for modules in [project] in [ModuleConfigurationMode.INTERACTIVE] mode. * * 1. Create instance with [create] * 2. Ask use to choose from [modulesDTO] @@ -83,13 +85,15 @@ internal class ModulesSdkConfigurator private constructor( .toList() } - companion object { + internal companion object { + /** - * Create instance and save in [project] + * Create instance and save in [project], see class doc */ - suspend fun create(project: Project): ModulesSdkConfigurator = ModulesSdkConfigurator(project, getModulesWithoutSDKCreateInfo(project), PathShortener.create(project)).also { - project.putUserData(key, it) - } + suspend fun create(project: Project): ModulesSdkConfigurator = + ModulesSdkConfigurator(project, getModulesWithoutSDKCreateInfo(project), PathShortener.create(project)).also { + project.putUserData(key, it) + } /** * Get instance from project and **clear it** @@ -101,49 +105,22 @@ internal class ModulesSdkConfigurator private constructor( return instance } - private suspend fun getModulesWithoutSDKCreateInfo(project: Project): Map = withBackgroundProgress(project, PySdkConfiguratorBundle.message("intellij.python.sdk.looking")) { - val tools = PyProjectSdkConfigurationExtension.createMap() - val now = System.currentTimeMillis() - val resultDef = project.modules.filter { PythonSdkUtil.findPythonSdk(it) == null }.map { module -> - async { - val moduleInfo = getModuleInfo(module, tools) ?: return@async null - Pair(module, moduleInfo) - } - } - val result = resultDef.awaitAll().filterNotNull() - logger.debug { "SDKs calculated in ${System.currentTimeMillis() - now}ms" } - result.associate { (module, createInfoAndDTO) -> - Pair(module.name, createInfoAndDTO) - } - } - - private val logger = fileLogger() - - private sealed interface ModuleCreateInfo { - data class CreateSdkInfoWrapper(val createSdkInfo: CreateSdkInfo, val toolId: ToolId, val moduleDir: Directory?) : ModuleCreateInfo - data class SameAs(val parentModuleName: ModuleName) : ModuleCreateInfo - } - - - private suspend fun getModuleInfo(module: Module, configuratorsByTool: Map): ModuleCreateInfo? = // Save on module level - when (val r = module.suggestSdk()) { - is SuggestedSdk.PyProjectIndependent -> { - val tools = r.preferTools.map { configuratorsByTool[it]!! } - tools.firstNotNullOfOrNull { tool -> - val createInfo = (tool.asPyProjectTomlSdkConfigurationExtension()?.createSdkWithoutPyProjectTomlChecks(module) - ?: tool.checkEnvironmentAndPrepareSdkCreator(module)) ?: return@firstNotNullOfOrNull null - CreateSdkInfoWithTool(createInfo, tool.toolId).asDTO(r.moduleDir) + private suspend fun getModulesWithoutSDKCreateInfo(project: Project): Map = + withBackgroundProgress(project, PySdkConfiguratorBundle.message("intellij.python.sdk.looking")) { + val tools = PyProjectSdkConfigurationExtension.createMap() + val now = System.currentTimeMillis() + val resultDef = project.modules.filter { PythonSdkUtil.findPythonSdk(it) == null }.map { module -> + async { + val moduleInfo = getModuleInfo(module, tools) ?: return@async null + Pair(module, moduleInfo) } } - is SuggestedSdk.SameAs -> { - ModuleCreateInfo.SameAs(r.parentModule.name) + val result = resultDef.awaitAll().filterNotNull() + logger.debug { "SDKs calculated in ${System.currentTimeMillis() - now}ms" } + result.associate { (module, createInfoAndDTO) -> + Pair(module.name, createInfoAndDTO) } - null -> null - } // No tools or not pyproject.toml at all? Use EP as a fallback - ?: PyProjectSdkConfigurationExtension.findAllSortedForModule(module).firstOrNull()?.let { CreateSdkInfoWithTool(it.createSdkInfo, it.toolId).asDTO(module.guessModuleDir()?.toNioPath()) } - - - private fun CreateSdkInfoWithTool.asDTO(moduleDir: Directory?): ModuleCreateInfo = ModuleCreateInfo.CreateSdkInfoWrapper(createSdkInfo, toolId, moduleDir) + } /** @@ -204,3 +181,90 @@ internal class ModulesSdkConfigurator private constructor( } } +/** + * See [ModuleConfigurationMode.AUTOMATIC] + */ +@ApiStatus.Internal // Opened for tests only: we can't put tests here because configurators are in communuty.impl +suspend fun configureSdkAutomatically(project: Project): Unit = withContext(Dispatchers.Default) { + val modules = project.modules + + when (modules.size) { + 0 -> return@withContext + 1 -> { // Single-module project, associate first module only + val module = modules.first() + if (PythonSdkUtil.findPythonSdk(module) != null) return@withContext // Already has SDK + when (val moduleInfo = getModuleInfo(module)) { + is ModuleCreateInfo.CreateSdkInfoWrapper -> { + when (val info = moduleInfo.createSdkInfo) { + is CreateSdkInfo.ExistingEnv -> { + when (val r = info.sdkCreator(false)) { + is Result.Failure -> { + logger.trace { "Failed to create sdk for ${module.name} : ${r.error}" } + } + is Result.Success -> { + val sdk = r.result!! // It can't be null: this is an old buggy API that will be fixed soon + ModuleRootModificationUtil.setModuleSdk(module, sdk) + logger.trace { "SDK creation result for ${module.name} : $sdk" } + } + } + } + is CreateSdkInfo.WillCreateEnv -> { + logger.trace { "${module.name} can't be configured automatically: no venv for ${info.intentionName}" } + } + } + } + is ModuleCreateInfo.SameAs, null -> Unit + } + } + else -> { // Multi-module project, only associate workspace members + val modulesToSdk = modules.filter { PythonSdkUtil.findPythonSdk(it) == null }.associateWith { it.suggestSdk() } + for ((module, sdkSuggestion) in modulesToSdk) { + when (sdkSuggestion) { + is SuggestedSdk.PyProjectIndependent, null -> { + logger.trace { "${module.name} skipped in multimodule project autoconfig" } + } + is SuggestedSdk.SameAs -> { + val parentSdk = PythonSdkUtil.findPythonSdk(sdkSuggestion.parentModule) ?: continue + logger.trace { "${module.name} seeds the same sdk as ${sdkSuggestion.parentModule} : ${parentSdk}" } + ModuleRootModificationUtil.setModuleSdk(module, parentSdk) + } + } + } + } + } +} + + +private sealed interface ModuleCreateInfo { + data class CreateSdkInfoWrapper(val createSdkInfo: CreateSdkInfo, val toolId: ToolId, val moduleDir: Directory?) : ModuleCreateInfo + data class SameAs(val parentModuleName: ModuleName) : ModuleCreateInfo +} + +private val logger = fileLogger() + +/** + * For multiple calls, pull [configuratorsByTool] up not to create it each time + */ +private suspend fun getModuleInfo( + module: Module, + configuratorsByTool: Map = PyProjectSdkConfigurationExtension.createMap(), +): ModuleCreateInfo? = // Save on module level + when (val r = module.suggestSdk()) { + is SuggestedSdk.PyProjectIndependent -> { + val tools = r.preferTools.map { configuratorsByTool[it]!! } + tools.firstNotNullOfOrNull { tool -> + val createInfo = (tool.asPyProjectTomlSdkConfigurationExtension()?.createSdkWithoutPyProjectTomlChecks(module) + ?: tool.checkEnvironmentAndPrepareSdkCreator(module)) ?: return@firstNotNullOfOrNull null + CreateSdkInfoWithTool(createInfo, tool.toolId).asDTO(r.moduleDir) + } + } + is SuggestedSdk.SameAs -> { + ModuleCreateInfo.SameAs(r.parentModule.name) + } + null -> null + } // No tools or not pyproject.toml at all? Use EP as a fallback + ?: PyProjectSdkConfigurationExtension.findAllSortedForModule(module).firstOrNull() + ?.let { CreateSdkInfoWithTool(it.createSdkInfo, it.toolId).asDTO(module.guessModuleDir()?.toNioPath()) } + +private fun CreateSdkInfoWithTool.asDTO(moduleDir: Directory?): ModuleCreateInfo = + ModuleCreateInfo.CreateSdkInfoWrapper(createSdkInfo, toolId, moduleDir) diff --git a/python/python-sdk-configurator/backend/src/impl/backendLib.kt b/python/python-sdk-configurator/backend/src/impl/backendLib.kt index 651ee4ca3020..46dc793283f5 100644 --- a/python/python-sdk-configurator/backend/src/impl/backendLib.kt +++ b/python/python-sdk-configurator/backend/src/impl/backendLib.kt @@ -14,28 +14,35 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -private val askUserMutex = Mutex() +private val configureMutex = Mutex() /** - * Same as [configureSdkAskingUser] but in a separate coroutine + * Same as [configureSdk] but in a separate coroutine */ -internal fun configureSdkAskingUserBg(project: Project) { +internal fun configureSdkBg(project: Project, mode: ModuleConfigurationMode) { project.service().scope.launch(Dispatchers.Default) { - configureSdkAskingUser(project) + configureSdk(project, mode) } } /** - * Ask user for list of modules and configure them + * Configure SDKs for [project] in [ModuleConfigurationMode] manner (see its doc for the semantics) */ -internal suspend fun configureSdkAskingUser(project: Project) { +internal suspend fun configureSdk(project: Project, mode: ModuleConfigurationMode) { withContext(Dispatchers.Default) { - askUserMutex.withLock { - val moduleToSuggestedSdk = ModulesSdkConfigurator.create(project) - val modulesDTO = moduleToSuggestedSdk.modulesDTO - if (modulesDTO.isNotEmpty()) { - // No need to send empty list - SHOW_SDK_CONFIG_UI_TOPIC.sendToClient(project, ModulesDTO(modulesDTO)) + configureMutex.withLock { + when (mode) { + ModuleConfigurationMode.AUTOMATIC -> { + configureSdkAutomatically(project) + } + ModuleConfigurationMode.INTERACTIVE -> { + val moduleToSuggestedSdk = ModulesSdkConfigurator.create(project) + val modulesDTO = moduleToSuggestedSdk.modulesDTO + if (modulesDTO.isNotEmpty()) { + // No need to send empty list + SHOW_SDK_CONFIG_UI_TOPIC.sendToClient(project, ModulesDTO(modulesDTO)) + } + } } } } diff --git a/python/python-sdk-configurator/backend/src/impl/platformBridge/ConfigureSDKAction.kt b/python/python-sdk-configurator/backend/src/impl/platformBridge/ConfigureSDKAction.kt index ebc8fd653827..64c0dc47af20 100644 --- a/python/python-sdk-configurator/backend/src/impl/platformBridge/ConfigureSDKAction.kt +++ b/python/python-sdk-configurator/backend/src/impl/platformBridge/ConfigureSDKAction.kt @@ -3,8 +3,8 @@ package com.intellij.python.sdkConfigurator.backend.impl.platformBridge import com.intellij.openapi.actionSystem.ActionUpdateThread import com.intellij.openapi.actionSystem.AnAction import com.intellij.openapi.actionSystem.AnActionEvent -import com.intellij.python.sdkConfigurator.backend.impl.configureSdkAskingUser -import com.intellij.python.sdkConfigurator.backend.impl.configureSdkAskingUserBg +import com.intellij.python.sdkConfigurator.backend.impl.ModuleConfigurationMode +import com.intellij.python.sdkConfigurator.backend.impl.configureSdkBg import com.intellij.python.sdkConfigurator.common.enableSDKAutoConfigurator internal class ConfigureSDKAction : AnAction() { @@ -14,7 +14,7 @@ internal class ConfigureSDKAction : AnAction() { if (!enableSDKAutoConfigurator) { return } - configureSdkAskingUserBg(project) + configureSdkBg(project, ModuleConfigurationMode.INTERACTIVE) } override fun update(e: AnActionEvent) { diff --git a/python/python-sdk-configurator/backend/src/impl/platformBridge/ModelRebuiltListenerImpl.kt b/python/python-sdk-configurator/backend/src/impl/platformBridge/ModelRebuiltListenerImplToConfigureSdk.kt similarity index 70% rename from python/python-sdk-configurator/backend/src/impl/platformBridge/ModelRebuiltListenerImpl.kt rename to python/python-sdk-configurator/backend/src/impl/platformBridge/ModelRebuiltListenerImplToConfigureSdk.kt index 41a9fdde637f..accba645ba55 100644 --- a/python/python-sdk-configurator/backend/src/impl/platformBridge/ModelRebuiltListenerImpl.kt +++ b/python/python-sdk-configurator/backend/src/impl/platformBridge/ModelRebuiltListenerImplToConfigureSdk.kt @@ -3,10 +3,11 @@ package com.intellij.python.sdkConfigurator.backend.impl.platformBridge import com.intellij.openapi.extensions.ExtensionNotApplicableException import com.intellij.openapi.project.Project import com.intellij.python.pyproject.model.api.ModelRebuiltListener -import com.intellij.python.sdkConfigurator.backend.impl.configureSdkAskingUserBg +import com.intellij.python.sdkConfigurator.backend.impl.ModuleConfigurationMode +import com.intellij.python.sdkConfigurator.backend.impl.configureSdkBg import com.intellij.python.sdkConfigurator.common.enableSDKAutoConfigurator -internal class ModelRebuiltListenerImpl : ModelRebuiltListener { +internal class ModelRebuiltListenerImplToConfigureSdk : ModelRebuiltListener { init { if (!enableSDKAutoConfigurator) { throw ExtensionNotApplicableException.create() @@ -14,7 +15,7 @@ internal class ModelRebuiltListenerImpl : ModelRebuiltListener { } override fun modelRebuilt(project: Project) { - configureSdkAskingUserBg(project) + configureSdkBg(project, mode = ModuleConfigurationMode.AUTOMATIC) } } diff --git a/python/python-sdk-configurator/backend/src/impl/rpcBridge/SdkConfiguratorApiImpl.kt b/python/python-sdk-configurator/backend/src/impl/rpcBridge/SdkConfiguratorApiImpl.kt index 557f7cc97e48..e8a0e205d35b 100644 --- a/python/python-sdk-configurator/backend/src/impl/rpcBridge/SdkConfiguratorApiImpl.kt +++ b/python/python-sdk-configurator/backend/src/impl/rpcBridge/SdkConfiguratorApiImpl.kt @@ -2,8 +2,9 @@ package com.intellij.python.sdkConfigurator.backend.impl.rpcBridge import com.intellij.platform.project.ProjectId import com.intellij.platform.project.findProject +import com.intellij.python.sdkConfigurator.backend.impl.ModuleConfigurationMode import com.intellij.python.sdkConfigurator.backend.impl.ModulesSdkConfigurator.Companion.popModulesSDKConfigurator -import com.intellij.python.sdkConfigurator.backend.impl.configureSdkAskingUser +import com.intellij.python.sdkConfigurator.backend.impl.configureSdk import com.intellij.python.sdkConfigurator.common.impl.ModuleName import com.intellij.python.sdkConfigurator.common.impl.SdkConfiguratorBackEndApi @@ -13,6 +14,6 @@ internal object SdkConfiguratorApiImpl : SdkConfiguratorBackEndApi { } override suspend fun configureAskingUser(projectId: ProjectId) { - configureSdkAskingUser(projectId.findProject()) + configureSdk(projectId.findProject(), mode = ModuleConfigurationMode.INTERACTIVE) } } \ No newline at end of file diff --git a/python/python-sdk/src/com/jetbrains/python/sdk/configuration/CreateSdkInfo.kt b/python/python-sdk/src/com/jetbrains/python/sdk/configuration/CreateSdkInfo.kt index 8c6f032f922a..1f44d9d6e25c 100644 --- a/python/python-sdk/src/com/jetbrains/python/sdk/configuration/CreateSdkInfo.kt +++ b/python/python-sdk/src/com/jetbrains/python/sdk/configuration/CreateSdkInfo.kt @@ -15,7 +15,7 @@ typealias EnvExists = Boolean sealed interface CreateSdkInfo : Comparable { @get:IntentionName val intentionName: String - val sdkCreator: suspend (NeedsConfirmation) -> PyResult + val sdkCreator: suspend (needsConfirmation: NeedsConfirmation) -> PyResult /** * Nullable SDK is only possible when we requested user confirmation but didn't get it. The idea behind this function is to provide @@ -38,12 +38,12 @@ sealed interface CreateSdkInfo : Comparable { data class ExistingEnv( val pythonInfo: PythonInfo, override val intentionName: String, - override val sdkCreator: suspend (NeedsConfirmation) -> PyResult, + override val sdkCreator: suspend (needsConfirmation: NeedsConfirmation) -> PyResult, ) : CreateSdkInfo data class WillCreateEnv( override val intentionName: String, - override val sdkCreator: suspend (NeedsConfirmation) -> PyResult, + override val sdkCreator: suspend (needsConfirmation: NeedsConfirmation) -> PyResult, ) : CreateSdkInfo } @@ -58,7 +58,7 @@ sealed interface EnvCheckerResult { // TODO: Make internal after we drop WSL sdk configurator suspend fun prepareSdkCreator( envChecker: suspend (CheckExistence) -> EnvCheckerResult, - sdkCreator: (EnvExists) -> (suspend (NeedsConfirmation) -> PyResult), + sdkCreator: (EnvExists) -> (suspend (needsConfirmation: NeedsConfirmation) -> PyResult), ): CreateSdkInfo? { var res = envChecker(true) return when (res) {