From 841056dcf2998b49bd16dd708b424e1d5d8a2c85 Mon Sep 17 00:00:00 2001 From: Timur Malanin Date: Wed, 19 Feb 2025 18:38:49 +0000 Subject: [PATCH] Simplified Package Specification Handling in `specForPackage`. Replaced direct repository handling in `CompositePythonPackageManager` with a new `CompositePythonRepositoryManager` class to unify repository operations. Streamlined package installation, update, and reload processes by removing redundant repository manager resets. Removed conditional logic differentiating between package managers in `specForPackage`. Now, it defaults to always returning `PythonSimplePackageSpecification`.That allows to avoid errors with new package managers. Merge-request: IJ-MR-154229 Merged-by: Timur Malanin GitOrigin-RevId: 2f5df050838607d53d96bf7190676d0e315e727f --- .../messages/PyBundle.properties | 6 + .../PythonPackageManagementServiceBridge.kt | 20 +-- .../conda/CompositePythonPackageManager.kt | 123 ++++++++++-------- .../conda/CompositePythonRepositoryManager.kt | 118 +++++++++++++++++ .../packaging/conda/CondaRepositoryManger.kt | 10 +- .../management/PythonRepositoryManager.kt | 2 +- .../pip/PipBasedRepositoryManager.kt | 2 +- .../PyPackagingToolWindowService.kt | 8 +- .../management/TestPythonRepositoryManager.kt | 2 +- 9 files changed, 208 insertions(+), 83 deletions(-) create mode 100644 python/src/com/jetbrains/python/packaging/conda/CompositePythonRepositoryManager.kt diff --git a/python/pluginResources/messages/PyBundle.properties b/python/pluginResources/messages/PyBundle.properties index 89d370905cd2..4419cae89a90 100644 --- a/python/pluginResources/messages/PyBundle.properties +++ b/python/pluginResources/messages/PyBundle.properties @@ -410,6 +410,12 @@ python.sdk.uv.executable.not.found=uv executable is not found python.sdk.uv.executable=uv executable: python.sdk.uv.install.packages.from.toml.checkbox.text=Install packages from pyproject.toml +# Composite package manager +python.packaging.composite.install.package.error=No suitable package managers were found to install the package: {0}. Tried the following managers: {1}. +python.packaging.composite.update.package.error=No suitable package managers were found to update the package: {0}. Tried the following managers: {1}. +python.packaging.composite.uninstall.package.error=No suitable package managers were found to uninstall the package: {0}. Tried the following managers: {1}. +python.packaging.composite.reload.packages.error=No suitable package managers were found to update the packages. Tried the following managers: {0}. + python.sdk.pipenv.has.been.selected=Pipenv interpreter has been already added, select ''{0}'' in your interpreters list python.sdk.there.is.no.interpreter=No interpreter python.sdk.no.interpreter.configured.warning=No Python interpreter configured for the project diff --git a/python/src/com/jetbrains/python/packaging/bridge/PythonPackageManagementServiceBridge.kt b/python/src/com/jetbrains/python/packaging/bridge/PythonPackageManagementServiceBridge.kt index 72e761f2794c..42b77a9ed04e 100644 --- a/python/src/com/jetbrains/python/packaging/bridge/PythonPackageManagementServiceBridge.kt +++ b/python/src/com/jetbrains/python/packaging/bridge/PythonPackageManagementServiceBridge.kt @@ -15,11 +15,13 @@ import com.intellij.webcore.packaging.RepoPackage import com.jetbrains.python.PyBundle import com.jetbrains.python.packaging.PyPackagingSettings import com.jetbrains.python.packaging.common.* -import com.jetbrains.python.packaging.conda.* +import com.jetbrains.python.packaging.conda.CondaPackage +import com.jetbrains.python.packaging.conda.CondaPackageCache +import com.jetbrains.python.packaging.conda.CondaPackageManager +import com.jetbrains.python.packaging.conda.CondaPackageRepository import com.jetbrains.python.packaging.management.PythonPackageManager import com.jetbrains.python.packaging.management.packagesByRepository import com.jetbrains.python.packaging.management.runPackagingTool -import com.jetbrains.python.packaging.pip.PipPythonPackageManager import com.jetbrains.python.packaging.repository.PyPIPackageRepository import com.jetbrains.python.packaging.repository.PyPackageRepository import com.jetbrains.python.packaging.ui.PyPackageManagementService @@ -105,7 +107,7 @@ class PythonPackageManagementServiceBridge(project: Project,sdk: Sdk) : PyPackag override fun reloadAllPackages(): List { return runBlocking { - manager.repositoryManager.refreshCashes() + manager.repositoryManager.refreshCaches() allPackages } } @@ -223,16 +225,8 @@ class PythonPackageManagementServiceBridge(project: Project,sdk: Sdk) : PyPackag } } - private fun specForPackage(packageName: String, version: String? = null, repository: PyPackageRepository? = null): PythonPackageSpecification { - return when(manager) { - is CondaPackageManager -> when { - useConda -> CondaPackageSpecification(packageName, version) - else -> PythonSimplePackageSpecification(packageName, version, repository ?: findRepositoryForPackage(packageName)) - } - is PipPythonPackageManager -> PythonSimplePackageSpecification(packageName, version, repository ?: findRepositoryForPackage(packageName)) - else -> error("Unknown package manager") - } - } + private fun specForPackage(packageName: String, version: String? = null, repository: PyPackageRepository? = null): PythonPackageSpecification = + PythonSimplePackageSpecification(packageName, version, repository ?: findRepositoryForPackage(packageName)) override fun shouldFetchLatestVersionsForOnlyInstalledPackages(): Boolean = !(isConda && useConda) diff --git a/python/src/com/jetbrains/python/packaging/conda/CompositePythonPackageManager.kt b/python/src/com/jetbrains/python/packaging/conda/CompositePythonPackageManager.kt index 4eda95eb6a0d..d34371a38658 100644 --- a/python/src/com/jetbrains/python/packaging/conda/CompositePythonPackageManager.kt +++ b/python/src/com/jetbrains/python/packaging/conda/CompositePythonPackageManager.kt @@ -3,90 +3,101 @@ package com.jetbrains.python.packaging.conda import com.intellij.openapi.project.Project import com.intellij.openapi.projectRoots.Sdk +import com.jetbrains.python.PyBundle import com.jetbrains.python.packaging.common.PythonPackage import com.jetbrains.python.packaging.common.PythonPackageSpecification import com.jetbrains.python.packaging.management.PythonPackageManager import com.jetbrains.python.packaging.management.PythonRepositoryManager -class CompositePythonPackageManager( +internal class CompositePythonPackageManager( project: Project, sdk: Sdk, private val managers: List, ) : PythonPackageManager(project, sdk) { + @Volatile override var installedPackages: List = emptyList() - // TODO: composite one - override var repositoryManager: PythonRepositoryManager = managers.first().repositoryManager + override var repositoryManager: PythonRepositoryManager = + CompositePythonRepositoryManager(project, sdk, managers.map { it.repositoryManager }) - private fun isInRepository(repositoryManager: PythonRepositoryManager, pkgName: String) = - repositoryManager.allPackages().contains(pkgName) + private val managerNames = managers.joinToString { it.javaClass.simpleName } override suspend fun installPackageCommand(specification: PythonPackageSpecification, options: List): Result { - val exceptionList = mutableListOf() - for (manager in managers) { - repositoryManager = manager.repositoryManager - installedPackages = manager.installedPackages - - if (!isInRepository(repositoryManager, specification.name)) continue - val executionResult = manager.installPackage(specification, options) - executionResult - .onSuccess { return Result.success(Unit) } - .onFailure { exceptionList.add(it) } - } - - return Result.failure(exceptionList.lastOrNull() ?: RuntimeException("No package managers found for package $specification")) + return processPackageOperation( + errorMessageKey = "python.packaging.composite.install.package.error", + operation = { it.installPackage(specification, options) }, + name = specification.name + ) } override suspend fun updatePackageCommand(specification: PythonPackageSpecification): Result { - val exceptionList = mutableListOf() - - for (manager in managers) { - repositoryManager = manager.repositoryManager - installedPackages = manager.installedPackages - - if (!isInRepository(repositoryManager, specification.name)) continue - val executionResult = manager.updatePackage(specification) - executionResult - .onSuccess { return Result.success(Unit) } - .onFailure { exceptionList.add(it) } - } - - return Result.failure(exceptionList.lastOrNull() ?: RuntimeException("No package managers found for package $specification")) + return processPackageOperation( + errorMessageKey = "python.packaging.composite.update.package.error", + operation = { it.updatePackage(specification) }, + name = specification.name + ) } override suspend fun uninstallPackageCommand(pkg: PythonPackage): Result { - val exceptionList = mutableListOf() - - for (manager in managers) { - repositoryManager = manager.repositoryManager - installedPackages = manager.installedPackages - - if (!isInRepository(repositoryManager, pkg.name)) { - continue - } - - val executionResult = manager.uninstallPackage(pkg) - executionResult - .onSuccess { return Result.success(Unit) } - .onFailure { exceptionList.add(it) } - } - - return Result.failure(exceptionList.lastOrNull() ?: RuntimeException("No package managers found for package $pkg")) + return processPackageOperation( + errorMessageKey = "python.packaging.composite.uninstall.package.error", + operation = { it.uninstallPackage(pkg) }, + name = pkg.name + ) } override suspend fun reloadPackagesCommand(): Result> { - val exceptionList = mutableListOf() + val results = mutableListOf() + val exceptions = mutableListOf() for (manager in managers) { - repositoryManager = manager.repositoryManager - installedPackages = manager.installedPackages - manager.reloadPackages() - .onSuccess { return Result.success(it) } - .onFailure { exceptionList.add(it) } + .onSuccess { results.addAll(it) } + .onFailure { exceptions.add(it) } } - return Result.failure(exceptionList.lastOrNull() ?: RuntimeException("No package managers found")) + return if (results.isNotEmpty()) { + Result.success(results) + } + else { + Result.failure(createCompositeException( + exceptions, + PyBundle.message("python.packaging.composite.reload.packages.error", managerNames) + )) + } + } + + private suspend fun processPackageOperation( + errorMessageKey: String, + operation: suspend (PythonPackageManager) -> Result>, + name: String, + ): Result { + val exceptions = mutableListOf() + for (manager in managers) { + operation(manager) + .onSuccess { return Result.success(Unit) } + .onFailure { exceptions.add(it) } + } + + return Result.failure(createCompositeException( + exceptions, + PyBundle.message(errorMessageKey, name, managerNames) + )) + } + + fun createCompositeException( + exceptions: List, + defaultMessage: String + ): RuntimeException { + if (exceptions.isEmpty()) { + return RuntimeException(defaultMessage) + } + + val concatenatedMessages = exceptions.joinToString(separator = "; ") { exception -> + exception.message ?: exception.toString() + } + + return RuntimeException(concatenatedMessages) } } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/packaging/conda/CompositePythonRepositoryManager.kt b/python/src/com/jetbrains/python/packaging/conda/CompositePythonRepositoryManager.kt new file mode 100644 index 000000000000..d7fe464e52fa --- /dev/null +++ b/python/src/com/jetbrains/python/packaging/conda/CompositePythonRepositoryManager.kt @@ -0,0 +1,118 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.jetbrains.python.packaging.conda + +import com.intellij.openapi.components.service +import com.intellij.openapi.project.Project +import com.intellij.openapi.projectRoots.Sdk +import com.jetbrains.python.PyBundle +import com.jetbrains.python.packaging.PyPackageVersion +import com.jetbrains.python.packaging.common.EmptyPythonPackageDetails +import com.jetbrains.python.packaging.common.PythonPackageDetails +import com.jetbrains.python.packaging.common.PythonPackageSpecification +import com.jetbrains.python.packaging.management.PythonPackageManagerService +import com.jetbrains.python.packaging.management.PythonRepositoryManager +import com.jetbrains.python.packaging.repository.PyPackageRepository +import io.github.z4kn4fein.semver.toVersion +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.sync.withPermit +import java.util.concurrent.atomic.AtomicBoolean + +internal class CompositePythonRepositoryManager( + project: Project, + sdk: Sdk, + private val managers: List, +) : PythonRepositoryManager(project, sdk) { + + override val repositories: List = + managers.flatMap { it.repositories } + + override fun allPackages(): List = + managers.flatMap { it.allPackages() } + + override fun packagesFromRepository(repository: PyPackageRepository): List { + return findPackagesInRepository(repository) + ?: error("No packages for requested repository in cache") + } + + private fun findPackagesInRepository(repository: PyPackageRepository): List? { + for (manager in managers) { + val packages = manager.packagesFromRepository(repository) + if (packages.isNotEmpty()) { + return packages + } + } + return null + } + + override suspend fun getPackageDetails(pkg: PythonPackageSpecification): PythonPackageDetails { + for (manager in managers) { + if (manager.allPackages().contains(pkg.name)) { + return manager.getPackageDetails(pkg) + } + } + return EmptyPythonPackageDetails(pkg.name, PyBundle.message("python.packaging.could.not.parse.response", pkg.name, pkg.repository?.name)) + } + + override suspend fun getLatestVersion(spec: PythonPackageSpecification): PyPackageVersion? { + var latestVersion: PyPackageVersion? = null + for (manager in managers) { + val version = manager.getLatestVersion(spec) + if (version != null && + (latestVersion == null || version.presentableText.toVersion() > latestVersion.presentableText.toVersion()) + ) { + latestVersion = version + } + } + return latestVersion + } + + private val mutex = Mutex() + private val isInit = AtomicBoolean(false) + private val cacheRefreshLimit = managers.size * 2 + private val cacheRefreshLimitSemaphore = Semaphore(cacheRefreshLimit) + + override suspend fun refreshCaches() { + mutex.withLock { + managers.forEach { manager -> + launchManagerRefresh(manager) + } + isInit.set(true) + } + } + + private fun launchManagerRefresh(manager: PythonRepositoryManager) { + project.service().getServiceScope().launch { + cacheRefreshLimitSemaphore.withPermit { + manager.refreshCaches() + } + } + } + + override suspend fun initCaches() { + if (!isInit.compareAndSet(false, true)) return + mutex.withLock { + managers.forEach { it.initCaches() } + } + } + + override fun buildPackageDetails(rawInfo: String?, spec: PythonPackageSpecification): PythonPackageDetails { + val repositoryWithPackage = managers.firstOrNull { it -> + it.allPackages().contains(spec.name) + } ?: error("No repository contains the package ${spec.name}") + + return repositoryWithPackage.buildPackageDetails(rawInfo, spec) + } + + override fun searchPackages(query: String, repository: PyPackageRepository): List { + return managers.flatMap { it.searchPackages(query, repository) } + } + + override fun searchPackages(query: String): Map> { + return managers.flatMap { it.searchPackages(query).entries } + .groupBy({ it.key }, { it.value }) + .mapValues { it.value.flatten() } + } +} \ No newline at end of file diff --git a/python/src/com/jetbrains/python/packaging/conda/CondaRepositoryManger.kt b/python/src/com/jetbrains/python/packaging/conda/CondaRepositoryManger.kt index 13b8fa203201..3258ef15fdb0 100644 --- a/python/src/com/jetbrains/python/packaging/conda/CondaRepositoryManger.kt +++ b/python/src/com/jetbrains/python/packaging/conda/CondaRepositoryManger.kt @@ -58,16 +58,16 @@ internal class CondaRepositoryManger(project: Project, sdk: Sdk) : PipBasedRepos return super.getLatestVersion(spec) } + override suspend fun refreshCaches() { + super.refreshCaches() + service().forceReloadCache(sdk, project) + } + override suspend fun initCaches() { super.initCaches() service().reloadCache(sdk, project) } - override suspend fun refreshCashes() { - super.refreshCashes() - service().forceReloadCache(sdk, project) - } - override fun searchPackages(query: String, repository: PyPackageRepository): List { return if (repository is CondaPackageRepository) { service().packages diff --git a/python/src/com/jetbrains/python/packaging/management/PythonRepositoryManager.kt b/python/src/com/jetbrains/python/packaging/management/PythonRepositoryManager.kt index cafadebfaf9a..047f6b4edc71 100644 --- a/python/src/com/jetbrains/python/packaging/management/PythonRepositoryManager.kt +++ b/python/src/com/jetbrains/python/packaging/management/PythonRepositoryManager.kt @@ -19,7 +19,7 @@ abstract class PythonRepositoryManager(val project: Project, val sdk: Sdk) { abstract suspend fun getPackageDetails(pkg: PythonPackageSpecification): PythonPackageDetails abstract suspend fun getLatestVersion(spec: PythonPackageSpecification): PyPackageVersion? - abstract suspend fun refreshCashes() + abstract suspend fun refreshCaches() abstract suspend fun initCaches() diff --git a/python/src/com/jetbrains/python/packaging/pip/PipBasedRepositoryManager.kt b/python/src/com/jetbrains/python/packaging/pip/PipBasedRepositoryManager.kt index f2b668b20ec5..965cc8cec14c 100644 --- a/python/src/com/jetbrains/python/packaging/pip/PipBasedRepositoryManager.kt +++ b/python/src/com/jetbrains/python/packaging/pip/PipBasedRepositoryManager.kt @@ -128,7 +128,7 @@ internal abstract class PipBasedRepositoryManager(project: Project, sdk: Sdk) : } } - override suspend fun refreshCashes() { + override suspend fun refreshCaches() { service().forceReloadCache() service().refresh() } diff --git a/python/src/com/jetbrains/python/packaging/toolwindow/PyPackagingToolWindowService.kt b/python/src/com/jetbrains/python/packaging/toolwindow/PyPackagingToolWindowService.kt index 48707c6a5af4..1d269ba47fcd 100644 --- a/python/src/com/jetbrains/python/packaging/toolwindow/PyPackagingToolWindowService.kt +++ b/python/src/com/jetbrains/python/packaging/toolwindow/PyPackagingToolWindowService.kt @@ -23,11 +23,7 @@ import com.intellij.platform.ide.progress.withBackgroundProgress import com.intellij.platform.util.progress.reportRawProgress import com.jetbrains.python.PyBundle.message import com.jetbrains.python.packaging.* -import com.jetbrains.python.packaging.common.PythonPackageDetails -import com.jetbrains.python.packaging.common.PythonPackageManagementListener -import com.jetbrains.python.packaging.common.PythonPackageSpecification -import com.jetbrains.python.packaging.common.normalizePackageName -import com.jetbrains.python.packaging.common.runPackagingOperationOrShowErrorDialog +import com.jetbrains.python.packaging.common.* import com.jetbrains.python.packaging.conda.CondaPackage import com.jetbrains.python.packaging.management.PythonPackageManager import com.jetbrains.python.packaging.management.packagesByRepository @@ -316,7 +312,7 @@ class PyPackagingToolWindowService(val project: Project, val serviceScope: Corou manager.reloadPackages() } refreshInstalledPackages() - manager.repositoryManager.refreshCashes() + manager.repositoryManager.refreshCaches() } } } diff --git a/python/testSrc/com/jetbrains/python/packaging/management/TestPythonRepositoryManager.kt b/python/testSrc/com/jetbrains/python/packaging/management/TestPythonRepositoryManager.kt index fb19f3a68f0d..3bf82e48aa98 100644 --- a/python/testSrc/com/jetbrains/python/packaging/management/TestPythonRepositoryManager.kt +++ b/python/testSrc/com/jetbrains/python/packaging/management/TestPythonRepositoryManager.kt @@ -58,7 +58,7 @@ class TestPythonRepositoryManager(project: Project, sdk: Sdk) : PythonRepository TODO("Not yet implemented") } - override suspend fun refreshCashes() { + override suspend fun refreshCaches() { } override suspend fun initCaches() {