From 0549092e0931ca74a327a6f99419f3d940f3845f Mon Sep 17 00:00:00 2001 From: "Ilya.Kazakevich" Date: Wed, 15 Oct 2025 19:06:13 +0200 Subject: [PATCH] [python] PY-79486: (WIP) Refactor to extract boilerplate from tools. No need to run the same project structure parsing logic again and again. GitOrigin-RevId: 6a3da2134f7124c361fa3e32e3925166c9b7d880 --- .../pyproject/model/internal/tomFileTools.kt | 54 ++++++++++++++++ .../model/internal/workspaceTools.kt | 7 +-- .../python/pyproject/model/spi/Tool.kt | 5 +- .../projectModel/common/PyProjectTomlUtils.kt | 63 ------------------- .../python/projectModel/hatch/HatchTool.kt | 6 +- .../python/projectModel/poetry/PoetryTool.kt | 19 +----- .../python/projectModel/uv/UvTool.kt | 7 ++- 7 files changed, 68 insertions(+), 93 deletions(-) delete mode 100644 python/src/com/jetbrains/python/projectModel/common/PyProjectTomlUtils.kt diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/tomFileTools.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/tomFileTools.kt index d4c1dc197a09..8584a6272ae2 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/tomFileTools.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/tomFileTools.kt @@ -3,15 +3,23 @@ package com.intellij.python.pyproject.model.internal import com.intellij.openapi.diagnostic.fileLogger import com.intellij.python.pyproject.PY_PROJECT_TOML import com.intellij.python.pyproject.PyProjectToml +import com.intellij.python.pyproject.model.spi.ProjectName +import com.intellij.python.pyproject.model.spi.ProjectStructureInfo +import com.intellij.python.pyproject.model.spi.PyProjectTomlProject +import com.intellij.util.concurrency.annotations.RequiresBackgroundThread import com.jetbrains.python.Result import com.jetbrains.python.venvReader.Directory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.IOException +import java.net.URI +import java.net.URISyntaxException import java.nio.file.FileVisitResult +import java.nio.file.InvalidPathException import java.nio.file.Path import kotlin.io.path.name import kotlin.io.path.readText +import kotlin.io.path.toPath import kotlin.io.path.visitFileTree // Tools to walk FS and parse pyproject.toml @@ -48,6 +56,23 @@ internal suspend fun walkFileSystem(root: Directory): FSWalkInfo { internal data class FSWalkInfo(val tomlFiles: Map, val excludeDir: Set) +internal suspend fun getProjectStructureDefault( + entries: Map, + rootIndex: Map, + // dependenciesGetter: (PyProjectTomlProject) -> Set, +): ProjectStructureInfo = withContext(Dispatchers.Default) { + val deps = entries.asSequence().associate { (name, entry) -> + val deps = getDependenciesFromToml(entry.pyProjectToml).mapNotNull { dir -> + rootIndex[dir] ?: run { + logger.warn("Can't find project for dir $dir") + null + } + }.toSet() + Pair(name, deps) + } + ProjectStructureInfo(dependencies = deps, membersToWorkspace = emptyMap()) // No workspace info (yet) +} + private val logger = fileLogger() private suspend fun readFile(file: Path): PyProjectToml? { @@ -66,3 +91,32 @@ private suspend fun readFile(file: Path): PyProjectToml? { is Result.Success -> r.result } } + +@RequiresBackgroundThread +private fun getDependenciesFromToml(projectToml: PyProjectToml): Set { + val depsFromFile = projectToml.project?.dependencies?.project ?: emptyList() + val moduleDependencies = depsFromFile + .mapNotNull { depSpec -> + val match = PEP_621_PATH_DEPENDENCY.matchEntire(depSpec) ?: return@mapNotNull null + val (_, depUri) = match.destructured + return@mapNotNull parseDepUri(depUri) + } + return moduleDependencies.toSet() +} + + +// e.g. "lib @ file:///home/user/projects/main/lib" +private val PEP_621_PATH_DEPENDENCY = """([\w-]+) @ (file:.*)""".toRegex() + +private fun parseDepUri(depUri: String): Path? = + try { + URI(depUri).toPath() + } + catch (e: InvalidPathException) { + logger.info("Dep $depUri points to wrong path", e) + null + } + catch (e: URISyntaxException) { + logger.info("Dep $depUri can't be parsed", e) + null + } \ No newline at end of file diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceTools.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceTools.kt index e6673d5708ba..3d3cf0cc1129 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceTools.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceTools.kt @@ -21,10 +21,7 @@ import com.intellij.platform.workspace.storage.url.VirtualFileUrl import com.intellij.platform.workspace.storage.url.VirtualFileUrlManager import com.intellij.python.pyproject.PyProjectToml import com.intellij.python.pyproject.model.api.ModelRebuiltListener -import com.intellij.python.pyproject.model.spi.ProjectName -import com.intellij.python.pyproject.model.spi.PyProjectTomlProject -import com.intellij.python.pyproject.model.spi.Tool -import com.intellij.python.pyproject.model.spi.WorkspaceName +import com.intellij.python.pyproject.model.spi.* import com.intellij.util.messages.Topic import com.jetbrains.python.ToolId import com.jetbrains.python.venvReader.Directory @@ -147,7 +144,7 @@ private suspend fun generatePyProjectTomlEntries(files: Map val namesByDir = entries.associate { Pair(it.root, it.name) } val allNames = entriesByName.keys for (tool in Tool.EP.extensionList) { - val (dependencies, workspaceMembers) = tool.getProjectStructure(entriesByName, namesByDir) + val (dependencies, workspaceMembers) = tool.getProjectStructure(entriesByName, namesByDir) ?: getProjectStructureDefault(entriesByName, namesByDir) for ((name, deps) in dependencies) { val orphanNames = deps - allNames assert(orphanNames.isEmpty()) { "Tool $tool retuned wrong project names ${orphanNames.joinToString(", ")}" } diff --git a/python/python-pyproject/src/com/intellij/python/pyproject/model/spi/Tool.kt b/python/python-pyproject/src/com/intellij/python/pyproject/model/spi/Tool.kt index 164b8e25b940..10b4819f2fee 100644 --- a/python/python-pyproject/src/com/intellij/python/pyproject/model/spi/Tool.kt +++ b/python/python-pyproject/src/com/intellij/python/pyproject/model/spi/Tool.kt @@ -22,9 +22,10 @@ interface Tool { /** * Tool uses [entries] ([rootIndex] contains the same data, used as index rooDir->project name) to report project dependencies and workspace members. - * All project names must be taken from provided data (use [rootIndex] to get name by directory) + * All project names must be taken from provided data (use [rootIndex] to get name by directory). + * If tool doesn't provide any specific structure (i.e: no dependencies except those described in pyproject.toml spec, no workspaces) return `null` */ - suspend fun getProjectStructure(entries: Map, rootIndex: Map): ProjectStructureInfo + suspend fun getProjectStructure(entries: Map, rootIndex: Map): ProjectStructureInfo? /** * Tool that supports build systems might return additional src directories diff --git a/python/src/com/jetbrains/python/projectModel/common/PyProjectTomlUtils.kt b/python/src/com/jetbrains/python/projectModel/common/PyProjectTomlUtils.kt deleted file mode 100644 index 616ff1e9c200..000000000000 --- a/python/src/com/jetbrains/python/projectModel/common/PyProjectTomlUtils.kt +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.jetbrains.python.projectModel.common - -import com.intellij.openapi.diagnostic.fileLogger -import com.intellij.python.pyproject.PyProjectToml -import com.intellij.python.pyproject.model.spi.ProjectName -import com.intellij.python.pyproject.model.spi.ProjectStructureInfo -import com.intellij.python.pyproject.model.spi.PyProjectTomlProject -import com.intellij.util.concurrency.annotations.RequiresBackgroundThread -import com.jetbrains.python.venvReader.Directory -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import java.net.URI -import java.net.URISyntaxException -import java.nio.file.InvalidPathException -import java.nio.file.Path -import kotlin.io.path.toPath - -@RequiresBackgroundThread -internal fun getDependenciesFromToml(projectToml: PyProjectToml): Set { - val depsFromFile = projectToml.project?.dependencies?.project ?: emptyList() - val moduleDependencies = depsFromFile - .mapNotNull { depSpec -> - val match = PEP_621_PATH_DEPENDENCY.matchEntire(depSpec) ?: return@mapNotNull null - val (_, depUri) = match.destructured - return@mapNotNull parseDepUri(depUri) - } - return moduleDependencies.toSet() -} - -internal suspend fun getProjectStructure( - entries: Map, - rootIndex: Map, - dependenciesGetter: (PyProjectTomlProject) -> Set, -): ProjectStructureInfo = withContext(Dispatchers.Default) { - val deps = entries.asSequence().map { (name, entry) -> - val deps = dependenciesGetter(entry).mapNotNull { dir -> - rootIndex[dir] ?: run { - logger.warn("Can't find project for dir $dir") - null - } - }.toSet() - Pair(name, deps) - }.toMap() - ProjectStructureInfo(dependencies = deps, membersToWorkspace = emptyMap()) // No workspace info (yet) -} - -// e.g. "lib @ file:///home/user/projects/main/lib" -private val PEP_621_PATH_DEPENDENCY = """([\w-]+) @ (file:.*)""".toRegex() - -private val logger = fileLogger() -internal fun parseDepUri(depUri: String): Path? = - try { - URI(depUri).toPath() - } - catch (e: InvalidPathException) { - logger.info("Dep $depUri points to wrong path", e) - null - } - catch (e: URISyntaxException) { - logger.info("Dep $depUri can't be parsed", e) - null - } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/projectModel/hatch/HatchTool.kt b/python/src/com/jetbrains/python/projectModel/hatch/HatchTool.kt index ae4b25151708..c3ec8165d1da 100644 --- a/python/src/com/jetbrains/python/projectModel/hatch/HatchTool.kt +++ b/python/src/com/jetbrains/python/projectModel/hatch/HatchTool.kt @@ -9,14 +9,13 @@ import com.intellij.python.pyproject.model.spi.Tool import com.intellij.python.sdk.ui.icons.PythonSdkUIIcons import com.jetbrains.python.PyToolUIInfo import com.jetbrains.python.ToolId -import com.jetbrains.python.projectModel.common.getDependenciesFromToml -import com.jetbrains.python.projectModel.common.getProjectStructure import com.jetbrains.python.venvReader.Directory import org.apache.tuweni.toml.TomlTable import org.jetbrains.annotations.ApiStatus @ApiStatus.Internal val HATCH_TOOL_ID: ToolId = ToolId("hatch") + internal class HatchTool : Tool { override val id: ToolId = HATCH_TOOL_ID @@ -26,6 +25,5 @@ internal class HatchTool : Tool { override suspend fun getProjectName(projectToml: TomlTable): @NlsSafe String? = null - override suspend fun getProjectStructure(entries: Map, rootIndex: Map): ProjectStructureInfo = - getProjectStructure(entries, rootIndex) { getDependenciesFromToml(it.pyProjectToml) } + override suspend fun getProjectStructure(entries: Map, rootIndex: Map): ProjectStructureInfo? = null } diff --git a/python/src/com/jetbrains/python/projectModel/poetry/PoetryTool.kt b/python/src/com/jetbrains/python/projectModel/poetry/PoetryTool.kt index 6bd5817da6cf..478177148711 100644 --- a/python/src/com/jetbrains/python/projectModel/poetry/PoetryTool.kt +++ b/python/src/com/jetbrains/python/projectModel/poetry/PoetryTool.kt @@ -2,23 +2,20 @@ package com.jetbrains.python.projectModel.poetry import com.intellij.openapi.util.NlsSafe -import com.intellij.python.pyproject.PyProjectToml import com.intellij.python.pyproject.model.spi.ProjectName import com.intellij.python.pyproject.model.spi.ProjectStructureInfo import com.intellij.python.pyproject.model.spi.PyProjectTomlProject import com.intellij.python.pyproject.model.spi.Tool import com.intellij.python.sdk.ui.icons.PythonSdkUIIcons -import com.intellij.util.concurrency.annotations.RequiresBackgroundThread import com.jetbrains.python.PyToolUIInfo import com.jetbrains.python.ToolId -import com.jetbrains.python.projectModel.common.getDependenciesFromToml -import com.jetbrains.python.projectModel.common.getProjectStructure import com.jetbrains.python.venvReader.Directory import org.apache.tuweni.toml.TomlTable import org.jetbrains.annotations.ApiStatus @ApiStatus.Internal val POETRY_TOOL_ID: ToolId = ToolId("poetry") + internal class PoetryTool : Tool { override val id: ToolId = POETRY_TOOL_ID @@ -29,18 +26,6 @@ internal class PoetryTool : Tool { override suspend fun getProjectName(projectToml: TomlTable): @NlsSafe String? = projectToml.getString("tool.poetry.name") - override suspend fun getProjectStructure(entries: Map, rootIndex: Map): ProjectStructureInfo = - getProjectStructure(entries, rootIndex) { getDependencies(it.root, it.pyProjectToml) } + override suspend fun getProjectStructure(entries: Map, rootIndex: Map): ProjectStructureInfo? = null - @RequiresBackgroundThread - private fun getDependencies(rootDir: Directory, projectToml: PyProjectToml): Set { - val moduleDependenciesSet = getDependenciesFromToml(projectToml) - val oldStyleModuleDependencies = projectToml.toml.getTableOrEmpty("tool.poetry.dependencies") - .toMap().entries - .mapNotNull { (_, depSpec) -> - if (depSpec !is TomlTable || depSpec.getBoolean("develop") != true) return@mapNotNull null - depSpec.getString("path")?.let { rootDir.resolve(it).normalize() } - } - return moduleDependenciesSet + oldStyleModuleDependencies.toSet() - } } diff --git a/python/src/com/jetbrains/python/projectModel/uv/UvTool.kt b/python/src/com/jetbrains/python/projectModel/uv/UvTool.kt index a041441a127d..4ec12cc634d4 100644 --- a/python/src/com/jetbrains/python/projectModel/uv/UvTool.kt +++ b/python/src/com/jetbrains/python/projectModel/uv/UvTool.kt @@ -4,8 +4,10 @@ package com.jetbrains.python.projectModel.uv import com.intellij.openapi.diagnostic.fileLogger import com.intellij.openapi.util.NlsSafe import com.intellij.openapi.util.getPathMatcher -import com.intellij.python.pyproject.PY_PROJECT_TOML_BUILD_SYSTEM -import com.intellij.python.pyproject.model.spi.* +import com.intellij.python.pyproject.model.spi.ProjectName +import com.intellij.python.pyproject.model.spi.ProjectStructureInfo +import com.intellij.python.pyproject.model.spi.PyProjectTomlProject +import com.intellij.python.pyproject.model.spi.Tool import com.intellij.util.concurrency.annotations.RequiresBackgroundThread import com.jetbrains.python.PyToolUIInfo import com.jetbrains.python.ToolId @@ -23,6 +25,7 @@ import kotlin.io.path.relativeTo @ApiStatus.Internal val UV_TOOL_ID: ToolId = ToolId("uv") + internal class UvTool : Tool { override val id: ToolId = UV_TOOL_ID