From 1480554414e33ca64c7dbf7092c43b6df8b91706 Mon Sep 17 00:00:00 2001 From: Nikita Ashihmin Date: Fri, 13 Jun 2025 15:38:02 +0000 Subject: [PATCH] PY-81864 PyPackages: Move saved requirements.txt path from module to sdk Signed-off-by: Nikita.Ashihmin Merge-request: IJ-MR-165494 Merged-by: Nikita Ashihmin GitOrigin-RevId: fc852bf8db5ebda9a138c2b7cc2f096143adce7b --- .../messages/PyBundle.properties | 9 +- .../python/sdk/PythonSdkAdditionalData.java | 44 ++++++- .../src/com/jetbrains/python/sdk/Sdks.kt | 13 ++ .../PyIntegratedToolsConfigurable.java | 44 +++++-- .../PyPackageRequirementsSettings.java | 38 ++++-- .../python/packaging/PyPackageUtil.java | 19 +-- .../python/packaging/PyRequirementsTxtUtil.kt | 64 ++++----- .../PythonPackageManagerServiceImpl.kt | 8 +- .../requirements/PythonRequirementTxtUtils.kt | 123 ++++++++++++++++++ .../PyPackageRequirementsInspectionTest.java | 2 + .../python/packaging/PyPackageUtilTest.java | 6 + .../packaging/PyRequirementsGenerationTest.kt | 16 ++- 12 files changed, 308 insertions(+), 78 deletions(-) create mode 100644 python/src/com/jetbrains/python/packaging/requirements/PythonRequirementTxtUtils.kt diff --git a/python/pluginResources/messages/PyBundle.properties b/python/pluginResources/messages/PyBundle.properties index 6abff674d3ca..f29c5e08fac6 100644 --- a/python/pluginResources/messages/PyBundle.properties +++ b/python/pluginResources/messages/PyBundle.properties @@ -716,7 +716,7 @@ code.insight.install.type.hints.action=Install stub packages code.insight.ignore.type.hints=Ignore code.insight.ignore.packages.qfix=Ignore {0,choice, 1#package|2#packages} configurable.pipenv.auto.detected=Auto-detected: {0} -configurable.choose.path.to.the.package.requirements.file=Choose path to the package requirements file: +configurable.choose.path.to.the.package.requirements.file=Choose path to the package requirements file for SDK: configurable.choose.working.directory=Choose the working directory: configurable.select.working.directory=Select Working Directory @@ -1065,7 +1065,7 @@ form.edit.smart.indent.pasted.lines=Smart indent pasted lines form.integrated.tools.analyze.python.code.in.docstrings=Analyze Python code in docstrings form.integrated.tools.default.test.runner=Default test runner: form.integrated.tools.docstring.format=Docstring format: -form.integrated.tools.package.requirements.file=Package requirements file: +form.integrated.tools.package.requirements.file=Package requirements file for SDK: form.integrated.tools.path.to.pipenv.executable=Path to Pipenv executable: form.integrated.tools.render.external.documentation.for.stdlib=Render external documentation for stdlib form.integrated.tools.sphinx.working.directory=Sphinx working directory: @@ -1238,10 +1238,11 @@ python.requirements.remove.unused=Remove unused requirements python.requirements.modify.base.files=Modify base files (defined with -r or --requirement) python.requirements.keep.matching.specifier=Keep existing version specifier if it matches the current version python.requirements.analyzing.imports.title=Analyzing Imports in Project -python.requirements.action.name=Sync Python requirements +python.requirements.action.name=Sync Python Requirements python.requirements.error.ends.with.slash=Error parsing requirements: file ends with '/' symbol. python.requirements.error.no.interpreter=Configured interpreter required to synchronize requirements. -python.requirements.warning.unhandled.lines=Could not analyze entries in requirements fie: {0} +python.requirements.warning.unhandled.lines=Could not analyze entries in requirements file: {0} +python.requirements.error.create.requirements.file=Could not create requirements.txt file python.requirements.info.file.ref.dropped=Some requirements from base files were not updated: {0} python.requirements.balloon=Sync Python requirements diff --git a/python/python-sdk/src/com/jetbrains/python/sdk/PythonSdkAdditionalData.java b/python/python-sdk/src/com/jetbrains/python/sdk/PythonSdkAdditionalData.java index 31a7d1f84280..7fdc4dc40ebf 100644 --- a/python/python-sdk/src/com/jetbrains/python/sdk/PythonSdkAdditionalData.java +++ b/python/python-sdk/src/com/jetbrains/python/sdk/PythonSdkAdditionalData.java @@ -2,6 +2,7 @@ package com.jetbrains.python.sdk; import com.google.gson.Gson; +import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.projectRoots.SdkAdditionalData; import com.intellij.openapi.util.JDOMExternalizer; @@ -22,6 +23,7 @@ import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.nio.file.Path; import java.util.*; /** @@ -29,6 +31,9 @@ import java.util.*; */ // TODO: Use new annotation-based API to save data instead of legacy manual save public class PythonSdkAdditionalData implements SdkAdditionalData { + @ApiStatus.Internal + public static final @NonNls String REQUIREMENT_TXT_DEFAULT = "requirements.txt"; + private static final @NonNls String PATHS_ADDED_BY_USER_ROOT = "PATHS_ADDED_BY_USER_ROOT"; private static final @NonNls String PATH_ADDED_BY_USER = "PATH_ADDED_BY_USER"; private static final @NonNls String PATHS_REMOVED_BY_USER_ROOT = "PATHS_REMOVED_BY_USER_ROOT"; @@ -36,6 +41,7 @@ public class PythonSdkAdditionalData implements SdkAdditionalData { private static final @NonNls String PATHS_TO_TRANSFER_ROOT = "PATHS_TO_TRANSFER_ROOT"; private static final @NonNls String PATH_TO_TRANSFER = "PATH_TO_TRANSFER"; private static final @NonNls String ASSOCIATED_PROJECT_PATH = "ASSOCIATED_PROJECT_PATH"; + private static final @NonNls String ASSOCIATED_REQUIRED_TXT_PATH = "ASSOCIATED_REQUIRED_TXT_PATH"; private static final @NonNls String SDK_UUID_FIELD_NAME = "SDK_UUID"; private static final @NonNls String FLAVOR_ID = "FLAVOR_ID"; @@ -48,6 +54,7 @@ public class PythonSdkAdditionalData implements SdkAdditionalData { private PyFlavorAndData myFlavorAndData; private String myAssociatedModulePath; + private Path myRequiredTxtPath; private final Gson myGson = new Gson(); @@ -86,6 +93,7 @@ public class PythonSdkAdditionalData implements SdkAdditionalData { myExcludedPaths = from.myExcludedPaths.clone(PythonPluginDisposable.getInstance()); myPathsToTransfer = from.myPathsToTransfer.clone(PythonPluginDisposable.getInstance()); myAssociatedModulePath = from.myAssociatedModulePath; + myRequiredTxtPath = from.myRequiredTxtPath; myFlavorAndData = from.myFlavorAndData; myUUID = from.myUUID; } @@ -138,7 +146,6 @@ public class PythonSdkAdditionalData implements SdkAdditionalData { return myAssociatedModulePath; } - /** * Be sure to use {@link com.intellij.openapi.projectRoots.SdkModificator} to save changes */ @@ -147,6 +154,27 @@ public class PythonSdkAdditionalData implements SdkAdditionalData { myAssociatedModulePath = modulePath == null ? null : FileUtil.toSystemIndependentName(modulePath); } + /** + * ONLY FOR INTERNAL USE! + * For external usage use {@link com.jetbrains.python.packaging.requirements.PyRequirementTxtUtils#findRequirementsTxt(Sdk)} + * Be sure to use {@link com.intellij.openapi.projectRoots.SdkModificator} to save changes + */ + @ApiStatus.Internal + public final Path getRequiredTxtPath() { + return myRequiredTxtPath; + } + + /** + * ONLY FOR INTERNAL USE! + * For external usage use {@link com.jetbrains.python.packaging.requirements.PyRequirementTxtUtils#saveRequirementsTxtPath(Project, Sdk, Path)} + * Be sure to use {@link com.intellij.openapi.projectRoots.SdkModificator} to save changes + */ + @ApiStatus.Internal + public final void setRequiredTxtPath(@Nullable Path requiredTxtPath) { + boolean isNotDefault = requiredTxtPath == null; + myRequiredTxtPath = isNotDefault ? null : requiredTxtPath; + } + public void save(final @NotNull Element rootElement) { savePaths(rootElement, myAddedPaths, PATHS_ADDED_BY_USER_ROOT, PATH_ADDED_BY_USER); savePaths(rootElement, myExcludedPaths, PATHS_REMOVED_BY_USER_ROOT, PATH_REMOVED_BY_USER); @@ -155,6 +183,11 @@ public class PythonSdkAdditionalData implements SdkAdditionalData { if (myAssociatedModulePath != null) { rootElement.setAttribute(ASSOCIATED_PROJECT_PATH, myAssociatedModulePath); } + + if (myRequiredTxtPath != null) { + rootElement.setAttribute(ASSOCIATED_REQUIRED_TXT_PATH, myRequiredTxtPath.toString()); + } + rootElement.setAttribute(SDK_UUID_FIELD_NAME, myUUID.toString()); JDOMExternalizer.write(rootElement, FLAVOR_ID, myFlavorAndData.getFlavor().getUniqueId()); JDOMExternalizer.write(rootElement, FLAVOR_DATA, myGson.toJson(myFlavorAndData.getData(), myFlavorAndData.getDataClass())); @@ -192,6 +225,15 @@ public class PythonSdkAdditionalData implements SdkAdditionalData { collectPaths(JDOMExternalizer.loadStringsList(element, PATHS_TO_TRANSFER_ROOT, PATH_TO_TRANSFER), myPathsToTransfer); if (element != null) { myAssociatedModulePath = element.getAttributeValue(ASSOCIATED_PROJECT_PATH); + + String storedRequiredTxtPath = element.getAttributeValue(ASSOCIATED_REQUIRED_TXT_PATH); + if (storedRequiredTxtPath != null) { + myRequiredTxtPath = Path.of(storedRequiredTxtPath); + } + else { + myRequiredTxtPath = null; + } + var uuidStr = element.getAttributeValue(SDK_UUID_FIELD_NAME); if (uuidStr != null) { myUUID = UUID.fromString(uuidStr); diff --git a/python/python-sdk/src/com/jetbrains/python/sdk/Sdks.kt b/python/python-sdk/src/com/jetbrains/python/sdk/Sdks.kt index c45caf18c6ab..cfcbc8682e82 100644 --- a/python/python-sdk/src/com/jetbrains/python/sdk/Sdks.kt +++ b/python/python-sdk/src/com/jetbrains/python/sdk/Sdks.kt @@ -235,4 +235,17 @@ suspend fun Sdk.setAssociationToPath(path: String?) { writeAction { modificator.commitChanges() } +} + +@ApiStatus.Internal +object PythonSdkAdditionalDataUtils { + @JvmStatic + fun associateSdkWithModulePath(sdk: Sdk, module: Module) { + val sdkModificator = sdk.sdkModificator + val additionalData = sdkModificator.sdkAdditionalData as? PythonSdkAdditionalData ?: return + additionalData.associatedModulePath = module.basePath ?: return + ApplicationManager.getApplication().runWriteAction { + sdkModificator.commitChanges() + } + } } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/configuration/PyIntegratedToolsConfigurable.java b/python/src/com/jetbrains/python/configuration/PyIntegratedToolsConfigurable.java index 7e93f1b5d5cb..ef69f3a42664 100644 --- a/python/src/com/jetbrains/python/configuration/PyIntegratedToolsConfigurable.java +++ b/python/src/com/jetbrains/python/configuration/PyIntegratedToolsConfigurable.java @@ -9,6 +9,7 @@ import com.intellij.facet.ui.FacetConfigurationQuickFix; import com.intellij.facet.ui.FacetEditorValidator; import com.intellij.facet.ui.ValidationResult; import com.intellij.ide.util.PropertiesComponent; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory; import com.intellij.openapi.fileTypes.PlainTextFileType; import com.intellij.openapi.module.Module; @@ -17,6 +18,7 @@ import com.intellij.openapi.options.SearchableConfigurable; import com.intellij.openapi.project.DefaultProjectFactory; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; +import com.intellij.openapi.projectRoots.SdkAdditionalData; import com.intellij.openapi.roots.ProjectRootManager; import com.intellij.openapi.ui.DialogPanel; import com.intellij.openapi.ui.TextFieldWithBrowseButton; @@ -39,8 +41,9 @@ import com.jetbrains.python.documentation.PyDocumentationSettings; import com.jetbrains.python.documentation.docstrings.DocStringFormat; import com.jetbrains.python.packaging.PyPackageManagerUI; import com.jetbrains.python.packaging.PyPackageRequirementsSettings; -import com.jetbrains.python.packaging.PyPackageUtil; import com.jetbrains.python.packaging.PyRequirementsKt; +import com.jetbrains.python.packaging.requirements.PythonRequirementTxtUtils; +import com.jetbrains.python.sdk.PythonSdkAdditionalData; import com.jetbrains.python.sdk.PythonSdkUtil; import com.jetbrains.python.sdk.pipenv.PipenvCommandExecutorKt; import com.jetbrains.python.testing.PyAbstractTestFactory; @@ -119,12 +122,36 @@ public class PyIntegratedToolsConfigurable implements SearchableConfigurable { } private @NotNull String getRequirementsPath() { - final String path = myPackagingSettings.getRequirementsPath(); - if (myModule != null && myPackagingSettings.isDefaultPath() && !PyPackageUtil.hasRequirementsTxt(myModule)) { + if (myModule == null) { return ""; } - else { - return path; + Sdk sdk = PythonSdkUtil.findPythonSdk(myModule); + if (sdk == null) { + return ""; + } + SdkAdditionalData data = sdk.getSdkAdditionalData(); + if (!(data instanceof PythonSdkAdditionalData)) { + return ""; + } + Path requiredTxtPath = ((PythonSdkAdditionalData)data).getRequiredTxtPath(); + final String path = requiredTxtPath != null ? requiredTxtPath.toString() : ""; + return path; + } + + private void setRequirementsPath(String requirementsPath) { + if (myModule == null) { + return; + } + + Sdk sdk = PythonSdkUtil.findPythonSdk(myModule); + if (sdk == null) { + return; + } + try { + PythonRequirementTxtUtils.saveRequirementsTxtPath(myModule.getProject(), sdk, Path.of(requirementsPath)); + } + catch (Throwable t) { + Logger.getInstance(PyIntegratedToolsConfigurable.class).warn("Failed to save requirements path", t); } } @@ -165,7 +192,7 @@ public class PyIntegratedToolsConfigurable implements SearchableConfigurable { } } }); - ui.install(Collections.singletonList(PyRequirementsKt.pyRequirement(name,null)), Collections.emptyList()); + ui.install(Collections.singletonList(PyRequirementsKt.pyRequirement(name, null)), Collections.emptyList()); } }; } @@ -225,7 +252,8 @@ public class PyIntegratedToolsConfigurable implements SearchableConfigurable { if (!getRequirementsPath().equals(myRequirementsPathField.getText())) { return true; } - if (!myPipEnvPathField.getText().equals(StringUtil.notNullize(PipenvCommandExecutorKt.getPipEnvPath(PropertiesComponent.getInstance())))) { + if (!myPipEnvPathField.getText() + .equals(StringUtil.notNullize(PipenvCommandExecutorKt.getPipEnvPath(PropertiesComponent.getInstance())))) { return true; } return ContainerUtil.exists(myCustomizePanels, panel -> panel.isModified()); @@ -256,7 +284,7 @@ public class PyIntegratedToolsConfigurable implements SearchableConfigurable { reparseFiles(Collections.singletonList(PlainTextFileType.INSTANCE.getDefaultExtension())); } myDocumentationSettings.setAnalyzeDoctest(analyzeDoctest.isSelected()); - myPackagingSettings.setRequirementsPath(myRequirementsPathField.getText()); + setRequirementsPath(myRequirementsPathField.getText()); DaemonCodeAnalyzer.getInstance(myProject).restart(); PipenvCommandExecutorKt.setPipEnvPath(PropertiesComponent.getInstance(), StringUtil.nullize(myPipEnvPathField.getText())); diff --git a/python/src/com/jetbrains/python/packaging/PyPackageRequirementsSettings.java b/python/src/com/jetbrains/python/packaging/PyPackageRequirementsSettings.java index 3b659d8fc480..91b12351400b 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackageRequirementsSettings.java +++ b/python/src/com/jetbrains/python/packaging/PyPackageRequirementsSettings.java @@ -5,16 +5,22 @@ import com.intellij.openapi.components.RoamingType; import com.intellij.openapi.components.State; import com.intellij.openapi.components.Storage; import com.intellij.openapi.module.Module; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.projectRoots.Sdk; import com.intellij.util.xmlb.annotations.OptionTag; import com.jetbrains.python.defaultProjectAwareService.PyDefaultProjectAwareModuleConfiguratorImpl; import com.jetbrains.python.defaultProjectAwareService.PyDefaultProjectAwareService; import com.jetbrains.python.defaultProjectAwareService.PyDefaultProjectAwareServiceClasses; import com.jetbrains.python.defaultProjectAwareService.PyDefaultProjectAwareServiceModuleConfigurator; +import com.jetbrains.python.sdk.PythonSdkAdditionalData; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -@ApiStatus.Internal +import java.nio.file.Path; + + +@ApiStatus.Internal public abstract class PyPackageRequirementsSettings extends PyDefaultProjectAwareService< PyPackageRequirementsSettings.ServiceState, PyPackageRequirementsSettings, @@ -22,20 +28,27 @@ public abstract class PyPackageRequirementsSettings extends PyDefaultProjectAwar PyPackageRequirementsSettings.ModuleService> { private static final PyDefaultProjectAwareServiceClasses< - ServiceState, - PyPackageRequirementsSettings, - AppService, - ModuleService> SERVICE_CLASSES = new PyDefaultProjectAwareServiceClasses<>(AppService.class, ModuleService.class); - private static final String DEFAULT_REQUIREMENTS_PATH = "requirements.txt"; + ServiceState, + PyPackageRequirementsSettings, + AppService, + ModuleService> SERVICE_CLASSES = new PyDefaultProjectAwareServiceClasses<>(AppService.class, ModuleService.class); protected PyPackageRequirementsSettings() { super(new ServiceState()); } + /** + * @deprecated Use {@link {@link com.jetbrains.python.packaging.requirements.PythonRequirementTxtUtils#findRequirementsTxt(Sdk)} instead. + */ + @Deprecated(forRemoval = true) public final @NotNull String getRequirementsPath() { return getState().myRequirementsPath; } + /** + * @deprecated Use {@link com.jetbrains.python.packaging.requirements.PythonRequirementTxtUtils#saveRequirementsTxtPath(Project, Sdk, Path)} instead. + */ + @Deprecated(forRemoval = true) public void setRequirementsPath(@NotNull String path) { getState().myRequirementsPath = path; } @@ -76,10 +89,6 @@ public abstract class PyPackageRequirementsSettings extends PyDefaultProjectAwar getState().myKeepMatchingSpecifier = forceUpdateVersionSpecifier; } - public final boolean isDefaultPath() { - return getRequirementsPath().equals(DEFAULT_REQUIREMENTS_PATH); - } - public static @NotNull PyPackageRequirementsSettings getInstance(@Nullable Module module) { return SERVICE_CLASSES.getService(module); } @@ -89,9 +98,14 @@ public abstract class PyPackageRequirementsSettings extends PyDefaultProjectAwar } public static final class ServiceState { - @OptionTag("requirementsPath") public @NotNull String myRequirementsPath = DEFAULT_REQUIREMENTS_PATH; + /** + * @deprecated Use {@link {@link com.jetbrains.python.packaging.requirements.PythonRequirementTxtUtils#findRequirementsTxt(Sdk)} instead. + */ + @Deprecated(forRemoval = true) + @OptionTag("requirementsPath") public @NotNull String myRequirementsPath = PythonSdkAdditionalData.REQUIREMENT_TXT_DEFAULT; - @OptionTag("versionSpecifier") public @NotNull PyRequirementsVersionSpecifierType myVersionSpecifier = PyRequirementsVersionSpecifierType.COMPATIBLE; + @OptionTag("versionSpecifier") public @NotNull PyRequirementsVersionSpecifierType myVersionSpecifier = + PyRequirementsVersionSpecifierType.COMPATIBLE; @OptionTag("removeUnused") public boolean myRemoveUnused = false; diff --git a/python/src/com/jetbrains/python/packaging/PyPackageUtil.java b/python/src/com/jetbrains/python/packaging/PyPackageUtil.java index 99595f922235..5242ea313057 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackageUtil.java +++ b/python/src/com/jetbrains/python/packaging/PyPackageUtil.java @@ -41,6 +41,7 @@ import com.jetbrains.python.PyNames; import com.jetbrains.python.PyPsiPackageUtil; import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; import com.jetbrains.python.codeInsight.typing.PyTypeShed; +import com.jetbrains.python.packaging.requirements.PythonRequirementTxtUtils; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyPsiUtils; import com.jetbrains.python.psi.resolve.PyResolveContext; @@ -111,21 +112,11 @@ public final class PyPackageUtil { } public static @Nullable VirtualFile findRequirementsTxt(@NotNull Module module) { - final String requirementsPath = PyPackageRequirementsSettings.getInstance(module).getRequirementsPath(); - if (!requirementsPath.isEmpty()) { - final VirtualFile file = LocalFileSystem.getInstance().findFileByPath(requirementsPath); - if (file != null) { - return file; - } - final ModuleRootManager manager = ModuleRootManager.getInstance(module); - for (VirtualFile root : manager.getContentRoots()) { - final VirtualFile fileInRoot = root.findFileByRelativePath(requirementsPath); - if (fileInRoot != null) { - return fileInRoot; - } - } + Sdk sdk = PythonSdkUtil.findPythonSdk(module); + if (sdk == null) { + return null; } - return null; + return PythonRequirementTxtUtils.findRequirementsTxt(sdk); } private static @Nullable PsiElement findSetupPyInstallRequires(@Nullable PyCallExpression setupCall) { diff --git a/python/src/com/jetbrains/python/packaging/PyRequirementsTxtUtil.kt b/python/src/com/jetbrains/python/packaging/PyRequirementsTxtUtil.kt index 9a7bfe746aa2..f5e35cedd6cc 100644 --- a/python/src/com/jetbrains/python/packaging/PyRequirementsTxtUtil.kt +++ b/python/src/com/jetbrains/python/packaging/PyRequirementsTxtUtil.kt @@ -7,6 +7,7 @@ import com.intellij.notification.NotificationGroupManager import com.intellij.notification.NotificationType import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.ReadAction +import com.intellij.openapi.application.runWriteAction import com.intellij.openapi.command.WriteCommandAction import com.intellij.openapi.fileChooser.FileChooserDescriptor import com.intellij.openapi.fileEditor.FileDocumentManager @@ -14,15 +15,12 @@ import com.intellij.openapi.fileTypes.FileTypeRegistry import com.intellij.openapi.module.Module import com.intellij.openapi.progress.ProgressIndicator import com.intellij.openapi.progress.Task -import com.intellij.openapi.progress.runBlockingCancellable -import com.intellij.openapi.progress.runBlockingMaybeCancellable import com.intellij.openapi.project.Project import com.intellij.openapi.project.rootManager import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.util.NlsContexts import com.intellij.openapi.util.Ref import com.intellij.openapi.util.text.StringUtil -import com.intellij.openapi.vfs.LocalFileSystem import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.PsiManager import com.intellij.ui.components.dialog @@ -33,13 +31,15 @@ import com.jetbrains.python.PySdkBundle import com.jetbrains.python.PythonFileType import com.jetbrains.python.packaging.common.PythonPackage import com.jetbrains.python.packaging.management.PythonPackageManager +import com.jetbrains.python.packaging.requirements.PythonRequirementTxtUtils import com.jetbrains.python.psi.PyFile import com.jetbrains.python.sdk.PySdkPopupFactory +import com.jetbrains.python.sdk.PythonSdkAdditionalData import com.jetbrains.python.sdk.PythonSdkUtil import com.jetbrains.python.util.runWithModalBlockingOrInBackground import org.jetbrains.annotations.ApiStatus -import java.nio.file.Paths import java.util.* +import kotlin.io.path.Path /** @@ -58,7 +58,7 @@ data class PyRequirementsAnalysisResult( val unchangedInBaseFiles: List, ) { companion object { - fun empty() = PyRequirementsAnalysisResult(emptyList(), emptyMap(), emptyList(), emptyList()) + fun empty(): PyRequirementsAnalysisResult = PyRequirementsAnalysisResult(emptyList(), emptyMap(), emptyList(), emptyList()) } fun withImportedPackages(importedPackages: MutableMap, settings: PyPackageRequirementsSettings): PyRequirementsAnalysisResult { @@ -111,35 +111,32 @@ internal fun syncWithImports(module: Module) { val settings = PyPackageRequirementsSettings.getInstance(module) if (!ApplicationManager.getApplication().isUnitTestMode) { - val proceed = showSyncSettingsDialog(module.project, settings) + val proceed = showSyncSettingsDialog(module.project, settings, sdk) if (!proceed) return } - var requirementsFile = PyPackageUtil.findRequirementsTxt(module) + val requirementsFile = PyPackageUtil.findRequirementsTxt(module) ?: runWriteAction { + PythonRequirementTxtUtils.createRequirementsTxtPath(module, sdk) + } + + if (requirementsFile == null) { + val text = PyBundle.message("python.requirements.error.create.requirements.file") + showNotification(notificationGroup, NotificationType.WARNING, text, module.project) + return + } + val matchResult = prepareRequirementsText(module, sdk, settings) - val psiManager = PsiManager.getInstance(module.project) WriteCommandAction.runWriteCommandAction(module.project, PyBundle.message("python.requirements.action.name"), null, { - if (requirementsFile == null) { - val path = Paths.get(settings.requirementsPath) - val location = when { - path.parent != null -> LocalFileSystem.getInstance().findFileByPath(path.parent.toString())!! - else -> module.rootManager.contentRoots.first() - } - val root = psiManager.findDirectory(location)!! - - var psiFile = root.findFile(path.fileName.toString()) - if (psiFile == null) psiFile = root.createFile(path.fileName.toString()) - - requirementsFile = psiFile.virtualFile - } val documentManager = FileDocumentManager.getInstance() - documentManager.getDocument(requirementsFile!!)!!.setText(matchResult.currentFileOutput.joinToString("\n")) + documentManager.getDocument(requirementsFile)!!.setText(matchResult.currentFileOutput.joinToString("\n")) matchResult.baseFilesOutput.forEach { (file, content) -> documentManager.getDocument(file)!!.setText(content.joinToString("\n")) } }) - psiManager.findFile(requirementsFile!!)?.navigate(true) + val psiManager = PsiManager.getInstance(module.project) + psiManager.findFile(requirementsFile)?.navigate(true) + if (matchResult.unhandledLines.isNotEmpty()) { val text = PyBundle.message("python.requirements.warning.unhandled.lines", matchResult.unhandledLines.joinToString(", ")) showNotification(notificationGroup, NotificationType.WARNING, text, module.project) @@ -196,15 +193,24 @@ private fun prepareRequirementsText(module: Module, sdk: Sdk, settings: PyPackag return analysisResult.withImportedPackages(importedPackages, settings) } -private fun showSyncSettingsDialog(project: Project, settings: PyPackageRequirementsSettings): Boolean { +private fun showSyncSettingsDialog(project: Project, settings: PyPackageRequirementsSettings, sdk: Sdk): Boolean { val ref = Ref.create(false) val descriptor = FileChooserDescriptor(true, false, false, false, false, false) val panel = panel { - row(PyBundle.message("form.integrated.tools.package.requirements.file")) { - textFieldWithBrowseButton(fileChooserDescriptor = descriptor) - .bindText(settings::getRequirementsPath, settings::setRequirementsPath) - .align(AlignX.FILL) - .focused() + val sdkAdditionalData = sdk.sdkAdditionalData as? PythonSdkAdditionalData + if (sdkAdditionalData != null) { + row(PyBundle.message("form.integrated.tools.package.requirements.file")) { + textFieldWithBrowseButton(fileChooserDescriptor = descriptor) + .bindText({ + sdkAdditionalData.requiredTxtPath?.toString() ?: "" + }, { stringPath -> + sdkAdditionalData.requiredTxtPath = runCatching { + stringPath.ifBlank { null }?.let { Path(it) } + }.getOrNull() + }) + .align(AlignX.FILL) + .focused() + } } row(PyBundle.message("python.requirements.version.label")) { comboBox(PyRequirementsVersionSpecifierType.entries) diff --git a/python/src/com/jetbrains/python/packaging/management/PythonPackageManagerServiceImpl.kt b/python/src/com/jetbrains/python/packaging/management/PythonPackageManagerServiceImpl.kt index 72e52c5d5367..eb47de8e7bbf 100644 --- a/python/src/com/jetbrains/python/packaging/management/PythonPackageManagerServiceImpl.kt +++ b/python/src/com/jetbrains/python/packaging/management/PythonPackageManagerServiceImpl.kt @@ -6,10 +6,11 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.util.Disposer import com.jetbrains.python.packaging.bridge.PythonPackageManagementServiceBridge +import com.jetbrains.python.packaging.requirements.PythonRequirementTxtUtils import com.jetbrains.python.sdk.PythonSdkAdditionalData import com.jetbrains.python.sdk.getOrCreateAdditionalData import kotlinx.coroutines.CoroutineScope -import java.util.UUID +import java.util.* import java.util.concurrent.ConcurrentHashMap internal class PythonPackageManagerServiceImpl(private val serviceScope: CoroutineScope) : PythonPackageManagerService, Disposable { @@ -24,8 +25,9 @@ internal class PythonPackageManagerServiceImpl(private val serviceScope: Corouti val cacheKey = (sdk.getOrCreateAdditionalData()).uuid return cache.computeIfAbsent(cacheKey) { - PythonPackageManagerProvider.EP_NAME.extensionList - .firstNotNullOf { it.createPackageManagerForSdk(project, sdk) } + val createdSdk = PythonPackageManagerProvider.EP_NAME.extensionList.firstNotNullOf { it.createPackageManagerForSdk(project, sdk) } + PythonRequirementTxtUtils.migrateRequirementsTxtPathFromModuleToSdk(project, sdk) + createdSdk } } diff --git a/python/src/com/jetbrains/python/packaging/requirements/PythonRequirementTxtUtils.kt b/python/src/com/jetbrains/python/packaging/requirements/PythonRequirementTxtUtils.kt new file mode 100644 index 000000000000..cd38548844d3 --- /dev/null +++ b/python/src/com/jetbrains/python/packaging/requirements/PythonRequirementTxtUtils.kt @@ -0,0 +1,123 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.jetbrains.python.packaging.requirements + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.runWriteAction +import com.intellij.openapi.application.writeAction +import com.intellij.openapi.diagnostic.thisLogger +import com.intellij.openapi.module.Module +import com.intellij.openapi.project.Project +import com.intellij.openapi.project.modules +import com.intellij.openapi.projectRoots.Sdk +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.openapi.vfs.VirtualFileManager +import com.intellij.openapi.vfs.ex.temp.TempFileSystem +import com.intellij.openapi.vfs.findOrCreateFile +import com.intellij.openapi.vfs.toNioPathOrNull +import com.jetbrains.python.packaging.PyPackageRequirementsSettings +import com.jetbrains.python.packaging.utils.PyPackageCoroutine +import com.jetbrains.python.sdk.PythonSdkAdditionalData +import com.jetbrains.python.sdk.PythonSdkAdditionalData.REQUIREMENT_TXT_DEFAULT +import com.jetbrains.python.sdk.associatedModulePath +import com.jetbrains.python.sdk.baseDir +import org.jetbrains.annotations.ApiStatus +import java.nio.file.Path + + +/** + * Migrate from the module persistent path to sdk path + */ +@ApiStatus.Internal +object PythonRequirementTxtUtils { + @JvmStatic + fun findRequirementsTxt(sdk: Sdk): VirtualFile? { + val data = sdk.getSdkAdditionalData() as? PythonSdkAdditionalData ?: return null + val requirementsPath = data.requiredTxtPath ?: Path.of(REQUIREMENT_TXT_DEFAULT) + if (requirementsPath.isAbsolute) { + return VirtualFileManager.getInstance().findFileByNioPath(requirementsPath) + } + val associatedModuleFile = data.associatedModuleVirtualFile ?: return null + return associatedModuleFile.findFileByRelativePath(requirementsPath.toString()) + } + + @JvmStatic + fun saveRequirementsTxtPath(project: Project, sdk: Sdk, path: Path) { + val sdkModificator = sdk.sdkModificator + val modifiedData = sdkModificator.sdkAdditionalData as? PythonSdkAdditionalData ?: return + + val associatedModulePath = sdk.associatedModulePath?.let { Path.of(it) } + val realPath = if (path.isAbsolute && associatedModulePath != null && path.startsWith(associatedModulePath)) { + associatedModulePath.relativize(path) + } + else { + path + } + + modifiedData.requiredTxtPath = realPath + if (ApplicationManager.getApplication().isDispatchThread) { + runWriteAction { + sdkModificator.commitChanges() + } + } + else { + PyPackageCoroutine.launch(project) { + writeAction { + sdkModificator.commitChanges() + } + } + } + } + + fun createRequirementsTxtPath(module: Module, sdk: Sdk): VirtualFile? { + val sdkAdditionalData = sdk.sdkAdditionalData as? PythonSdkAdditionalData ?: return null + val basePathString = sdkAdditionalData.associatedModuleVirtualFile ?: module.baseDir ?: return null + val requirementsFile = basePathString.findOrCreateFile(REQUIREMENT_TXT_DEFAULT) + + //Need to pass test, because TempFS doesn't support getNioPath() + val requirementFilePath = requirementsFile.toNioPathOrNull() ?: Path.of(requirementsFile.path) + saveRequirementsTxtPath(module.project, sdk, requirementFilePath) + + return requirementsFile + } + + + fun migrateRequirementsTxtPathFromModuleToSdk(project: Project, sdk: Sdk) { + val sdkAdditionalData = sdk.sdkAdditionalData as? PythonSdkAdditionalData ?: return + val newPath = sdkAdditionalData.requiredTxtPath + if (newPath != null) + return + + val originalPath = project.modules.firstNotNullOfOrNull { + getRequirementsTxtFromModule(it) + } ?: return + + val path = try { + Path.of(originalPath) + } + catch (t: Throwable) { + thisLogger().warn(t) + return + } + + saveRequirementsTxtPath(project, sdk, path) + } + + @Suppress("DEPRECATION", "removal") + private fun getRequirementsTxtFromModule(module: Module): String? { + val settings = PyPackageRequirementsSettings.getInstance(module) + + val requirementsPath = settings.state.myRequirementsPath + settings.state.myRequirementsPath = "" + return if (requirementsPath.isNotBlank() && requirementsPath != REQUIREMENT_TXT_DEFAULT) + requirementsPath + else + null + } + + private val PythonSdkAdditionalData.associatedModuleVirtualFile: VirtualFile? + get() { + val associatedModulePath = associatedModulePath ?: return null + val nioPath = Path.of(associatedModulePath) + return VirtualFileManager.getInstance().findFileByNioPath(nioPath) ?: TempFileSystem.getInstance().findFileByNioFile(nioPath) + } +} \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java b/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java index d14d5b925190..0bf7e1ec2a0f 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java @@ -8,6 +8,7 @@ import com.jetbrains.python.fixtures.PyInspectionTestCase; import com.jetbrains.python.packaging.PyRequirement; import com.jetbrains.python.packaging.common.PythonPackage; import com.jetbrains.python.psi.LanguageLevel; +import com.jetbrains.python.sdk.PythonSdkAdditionalDataUtils; import com.jetbrains.python.sdk.PythonSdkUtil; import com.jetbrains.python.sdk.pipenv.PipenvFilesUtilsKt; import org.jetbrains.annotations.NotNull; @@ -30,6 +31,7 @@ public class PyPackageRequirementsInspectionTest extends PyInspectionTestCase { public void setUp() throws Exception { super.setUp(); final Sdk sdk = PythonSdkUtil.findPythonSdk(myFixture.getModule()); + PythonSdkAdditionalDataUtils.associateSdkWithModulePath(sdk, myFixture.getModule()); assertNotNull(sdk); replacePythonPackageManagerServiceWithTestInstance(myFixture.getProject(), List.of()); } diff --git a/python/testSrc/com/jetbrains/python/packaging/PyPackageUtilTest.java b/python/testSrc/com/jetbrains/python/packaging/PyPackageUtilTest.java index 78129ebc1daf..e13e0eba904f 100644 --- a/python/testSrc/com/jetbrains/python/packaging/PyPackageUtilTest.java +++ b/python/testSrc/com/jetbrains/python/packaging/PyPackageUtilTest.java @@ -18,12 +18,15 @@ package com.jetbrains.python.packaging; import com.google.common.collect.ImmutableMap; import com.intellij.openapi.command.WriteCommandAction; import com.intellij.openapi.module.Module; +import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.vfs.VirtualFile; import com.jetbrains.python.fixtures.PyTestCase; import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.psi.PyCallExpression; import com.jetbrains.python.psi.PyExpression; import com.jetbrains.python.psi.PyFile; +import com.jetbrains.python.sdk.PythonSdkAdditionalDataUtils; +import com.jetbrains.python.sdk.PythonSdkUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -39,6 +42,9 @@ public class PyPackageUtilTest extends PyTestCase { super.setUp(); myFixture.copyDirectoryToProject("packaging/PyPackageUtil/" + getTestName(false), ""); + Module module = myFixture.getModule(); + Sdk sdk = PythonSdkUtil.findPythonSdk(module); + PythonSdkAdditionalDataUtils.associateSdkWithModulePath(sdk, module); } public void testAbsentSetupPyReading() { diff --git a/python/testSrc/com/jetbrains/python/packaging/PyRequirementsGenerationTest.kt b/python/testSrc/com/jetbrains/python/packaging/PyRequirementsGenerationTest.kt index 66a5dcf284b5..ae3b7b24fb2c 100644 --- a/python/testSrc/com/jetbrains/python/packaging/PyRequirementsGenerationTest.kt +++ b/python/testSrc/com/jetbrains/python/packaging/PyRequirementsGenerationTest.kt @@ -13,10 +13,11 @@ import com.intellij.testFramework.registerServiceInstance import com.jetbrains.python.fixtures.PyTestCase import com.jetbrains.python.packaging.common.PythonPackage import com.jetbrains.python.packaging.management.TestPythonPackageManagerService +import com.jetbrains.python.sdk.PythonSdkAdditionalDataUtils +import com.jetbrains.python.sdk.PythonSdkUtil import org.easymock.EasyMock class PyRequirementsGenerationTest : PyTestCase() { - private var oldPackageManagers: PyPackageManagers? = null private val installedPackages = mapOf("Django" to "3.0.0", "requests" to "2.22.0", @@ -54,8 +55,11 @@ class PyRequirementsGenerationTest : PyTestCase() { modifyBaseFiles: Boolean = false, packages: Map = installedPackages, ) { + val module = myFixture.getModule() + val sdk = PythonSdkUtil.findPythonSdk(module) + PythonSdkAdditionalDataUtils.associateSdkWithModulePath(sdk!!, module) val settings = PyPackageRequirementsSettings.getInstance(myFixture.module) - val oldRequirementsPath = settings.requirementsPath + val oldVersionSpecifier = settings.versionSpecifier val oldRemoveUnused = settings.removeUnused val oldModifyBaseFiles = settings.modifyBaseFiles @@ -63,27 +67,25 @@ class PyRequirementsGenerationTest : PyTestCase() { try { overrideInstalledPackages(packages) - settings.requirementsPath = "requirements.txt" settings.versionSpecifier = versionSpecifier settings.removeUnused = removeUnused settings.modifyBaseFiles = modifyBaseFiles val testName = getTestName(true) myFixture.copyDirectoryToProject(testName, "") - myFixture.configureFromTempProjectFile(settings.requirementsPath) + myFixture.configureFromTempProjectFile("requirements.txt") val action = ActionManager.getInstance().getAction("PySyncPythonRequirements") val context = SimpleDataContext.getSimpleContext(PlatformCoreDataKeys.MODULE, myFixture.module, (myFixture.editor as EditorEx).dataContext) val event = AnActionEvent.createFromAnAction(action, null, "", context) action.actionPerformed(event) - myFixture.checkResultByFile("$testName/new_${settings.requirementsPath}", true) + myFixture.checkResultByFile("$testName/new_requirements.txt", true) if (modifyBaseFiles) { - myFixture.checkResultByFile("base_${settings.requirementsPath}", "$testName/new_base_${settings.requirementsPath}", true) + myFixture.checkResultByFile("base_requirements.txt", "$testName/new_base_requirements.txt", true) } assertProjectFilesNotParsed(myFixture.file) } finally { - settings.requirementsPath = oldRequirementsPath settings.versionSpecifier = oldVersionSpecifier settings.removeUnused = oldRemoveUnused settings.modifyBaseFiles = oldModifyBaseFiles