[python] PY-79486: (WIP) Refactor to extract boilerplate from tools.

No need to run the same project structure parsing logic again and again.

GitOrigin-RevId: 6a3da2134f7124c361fa3e32e3925166c9b7d880

python/python-pyproject/src/com/intellij/python/pyproject/model/internal/tomFileTools.kt

@@ -3,15 +3,23 @@ package com.intellij.python.pyproject.model.internal
 import com.intellij.openapi.diagnostic.fileLogger
 import com.intellij.python.pyproject.PY_PROJECT_TOML
 import com.intellij.python.pyproject.PyProjectToml
+import com.intellij.python.pyproject.model.spi.ProjectName
+import com.intellij.python.pyproject.model.spi.ProjectStructureInfo
+import com.intellij.python.pyproject.model.spi.PyProjectTomlProject
+import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
 import com.jetbrains.python.Result
 import com.jetbrains.python.venvReader.Directory
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.io.IOException
+import java.net.URI
+import java.net.URISyntaxException
 import java.nio.file.FileVisitResult
+import java.nio.file.InvalidPathException
 import java.nio.file.Path
 import kotlin.io.path.name
 import kotlin.io.path.readText
+import kotlin.io.path.toPath
 import kotlin.io.path.visitFileTree
 
 // Tools to walk FS and parse pyproject.toml
@@ -48,6 +56,23 @@ internal suspend fun walkFileSystem(root: Directory): FSWalkInfo {
 
 internal data class FSWalkInfo(val tomlFiles: Map<Path, PyProjectToml>, val excludeDir: Set<Directory>)
 
+internal suspend fun getProjectStructureDefault(
+  entries: Map<ProjectName, PyProjectTomlProject>,
+  rootIndex: Map<Directory, ProjectName>,
+  //  dependenciesGetter: (PyProjectTomlProject) -> Set<Directory>,
+): ProjectStructureInfo = withContext(Dispatchers.Default) {
+  val deps = entries.asSequence().associate { (name, entry) ->
+    val deps = getDependenciesFromToml(entry.pyProjectToml).mapNotNull { dir ->
+      rootIndex[dir] ?: run {
+        logger.warn("Can't find project for dir $dir")
+        null
+      }
+    }.toSet()
+    Pair(name, deps)
+  }
+  ProjectStructureInfo(dependencies = deps, membersToWorkspace = emptyMap()) // No workspace info (yet)
+}
+
 private val logger = fileLogger()
 
 private suspend fun readFile(file: Path): PyProjectToml? {
@@ -66,3 +91,32 @@ private suspend fun readFile(file: Path): PyProjectToml? {
     is Result.Success -> r.result
   }
 }
+
+@RequiresBackgroundThread
+private fun getDependenciesFromToml(projectToml: PyProjectToml): Set<Directory> {
+  val depsFromFile = projectToml.project?.dependencies?.project ?: emptyList()
+  val moduleDependencies = depsFromFile
+    .mapNotNull { depSpec ->
+      val match = PEP_621_PATH_DEPENDENCY.matchEntire(depSpec) ?: return@mapNotNull null
+      val (_, depUri) = match.destructured
+      return@mapNotNull parseDepUri(depUri)
+    }
+  return moduleDependencies.toSet()
+}
+
+
+// e.g. "lib @ file:///home/user/projects/main/lib"
+private val PEP_621_PATH_DEPENDENCY = """([\w-]+) @ (file:.*)""".toRegex()
+
+private fun parseDepUri(depUri: String): Path? =
+  try {
+    URI(depUri).toPath()
+  }
+  catch (e: InvalidPathException) {
+    logger.info("Dep $depUri points to wrong path", e)
+    null
+  }
+  catch (e: URISyntaxException) {
+    logger.info("Dep $depUri can't be parsed", e)
+    null
+  }

python/python-pyproject/src/com/intellij/python/pyproject/model/internal/workspaceTools.kt

@@ -21,10 +21,7 @@ import com.intellij.platform.workspace.storage.url.VirtualFileUrl
 import com.intellij.platform.workspace.storage.url.VirtualFileUrlManager
 import com.intellij.python.pyproject.PyProjectToml
 import com.intellij.python.pyproject.model.api.ModelRebuiltListener
-import com.intellij.python.pyproject.model.spi.ProjectName
-import com.intellij.python.pyproject.model.spi.PyProjectTomlProject
-import com.intellij.python.pyproject.model.spi.Tool
-import com.intellij.python.pyproject.model.spi.WorkspaceName
+import com.intellij.python.pyproject.model.spi.*
 import com.intellij.util.messages.Topic
 import com.jetbrains.python.ToolId
 import com.jetbrains.python.venvReader.Directory
@@ -147,7 +144,7 @@ private suspend fun generatePyProjectTomlEntries(files: Map<Path, PyProjectToml>
   val namesByDir = entries.associate { Pair(it.root, it.name) }
   val allNames = entriesByName.keys
   for (tool in Tool.EP.extensionList) {
-    val (dependencies, workspaceMembers) = tool.getProjectStructure(entriesByName, namesByDir)
+    val (dependencies, workspaceMembers) = tool.getProjectStructure(entriesByName, namesByDir) ?: getProjectStructureDefault(entriesByName, namesByDir)
     for ((name, deps) in dependencies) {
       val orphanNames = deps - allNames
       assert(orphanNames.isEmpty()) { "Tool $tool retuned wrong project names ${orphanNames.joinToString(", ")}" }

python/python-pyproject/src/com/intellij/python/pyproject/model/spi/Tool.kt

@@ -22,9 +22,10 @@ interface Tool {
 
   /**
    * Tool uses [entries] ([rootIndex] contains the same data, used as index rooDir->project name) to report project dependencies and workspace members.
-   * All project names must be taken from provided data (use [rootIndex] to get name by directory)
+   * All project names must be taken from provided data (use [rootIndex] to get name by directory).
+   * If tool doesn't provide any specific structure (i.e: no dependencies except those described in pyproject.toml spec, no workspaces) return `null`
    */
-  suspend fun getProjectStructure(entries: Map<ProjectName, PyProjectTomlProject>, rootIndex: Map<Directory, ProjectName>): ProjectStructureInfo
+  suspend fun getProjectStructure(entries: Map<ProjectName, PyProjectTomlProject>, rootIndex: Map<Directory, ProjectName>): ProjectStructureInfo?
 
   /**
    * Tool that supports build systems might return additional src directories

python/src/com/jetbrains/python/projectModel/common/PyProjectTomlUtils.kt

@@ -1,63 +0,0 @@
-// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.projectModel.common
-
-import com.intellij.openapi.diagnostic.fileLogger
-import com.intellij.python.pyproject.PyProjectToml
-import com.intellij.python.pyproject.model.spi.ProjectName
-import com.intellij.python.pyproject.model.spi.ProjectStructureInfo
-import com.intellij.python.pyproject.model.spi.PyProjectTomlProject
-import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
-import com.jetbrains.python.venvReader.Directory
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.withContext
-import java.net.URI
-import java.net.URISyntaxException
-import java.nio.file.InvalidPathException
-import java.nio.file.Path
-import kotlin.io.path.toPath
-
-@RequiresBackgroundThread
-internal fun getDependenciesFromToml(projectToml: PyProjectToml): Set<Directory> {
-  val depsFromFile = projectToml.project?.dependencies?.project ?: emptyList()
-  val moduleDependencies = depsFromFile
-    .mapNotNull { depSpec ->
-      val match = PEP_621_PATH_DEPENDENCY.matchEntire(depSpec) ?: return@mapNotNull null
-      val (_, depUri) = match.destructured
-      return@mapNotNull parseDepUri(depUri)
-    }
-  return moduleDependencies.toSet()
-}
-
-internal suspend fun getProjectStructure(
-  entries: Map<ProjectName, PyProjectTomlProject>,
-  rootIndex: Map<Directory, ProjectName>,
-  dependenciesGetter: (PyProjectTomlProject) -> Set<Directory>,
-): ProjectStructureInfo = withContext(Dispatchers.Default) {
-  val deps = entries.asSequence().map { (name, entry) ->
-    val deps = dependenciesGetter(entry).mapNotNull { dir ->
-      rootIndex[dir] ?: run {
-        logger.warn("Can't find project for dir $dir")
-        null
-      }
-    }.toSet()
-    Pair(name, deps)
-  }.toMap()
-  ProjectStructureInfo(dependencies = deps, membersToWorkspace = emptyMap()) // No workspace info (yet)
-}
-
-// e.g. "lib @ file:///home/user/projects/main/lib"
-private val PEP_621_PATH_DEPENDENCY = """([\w-]+) @ (file:.*)""".toRegex()
-
-private val logger = fileLogger()
-internal fun parseDepUri(depUri: String): Path? =
-  try {
-    URI(depUri).toPath()
-  }
-  catch (e: InvalidPathException) {
-    logger.info("Dep $depUri points to wrong path", e)
-    null
-  }
-  catch (e: URISyntaxException) {
-    logger.info("Dep $depUri can't be parsed", e)
-    null
-  }

python/src/com/jetbrains/python/projectModel/hatch/HatchTool.kt

@@ -9,14 +9,13 @@ import com.intellij.python.pyproject.model.spi.Tool
 import com.intellij.python.sdk.ui.icons.PythonSdkUIIcons
 import com.jetbrains.python.PyToolUIInfo
 import com.jetbrains.python.ToolId
-import com.jetbrains.python.projectModel.common.getDependenciesFromToml
-import com.jetbrains.python.projectModel.common.getProjectStructure
 import com.jetbrains.python.venvReader.Directory
 import org.apache.tuweni.toml.TomlTable
 import org.jetbrains.annotations.ApiStatus
 
 @ApiStatus.Internal
 val HATCH_TOOL_ID: ToolId = ToolId("hatch")
+
 internal class HatchTool : Tool {
 
   override val id: ToolId = HATCH_TOOL_ID
@@ -26,6 +25,5 @@ internal class HatchTool : Tool {
 
   override suspend fun getProjectName(projectToml: TomlTable): @NlsSafe String? = null
 
-  override suspend fun getProjectStructure(entries: Map<ProjectName, PyProjectTomlProject>, rootIndex: Map<Directory, ProjectName>): ProjectStructureInfo =
-    getProjectStructure(entries, rootIndex) { getDependenciesFromToml(it.pyProjectToml) }
+  override suspend fun getProjectStructure(entries: Map<ProjectName, PyProjectTomlProject>, rootIndex: Map<Directory, ProjectName>): ProjectStructureInfo? = null
 }

python/src/com/jetbrains/python/projectModel/poetry/PoetryTool.kt

@@ -2,23 +2,20 @@
 package com.jetbrains.python.projectModel.poetry
 
 import com.intellij.openapi.util.NlsSafe
-import com.intellij.python.pyproject.PyProjectToml
 import com.intellij.python.pyproject.model.spi.ProjectName
 import com.intellij.python.pyproject.model.spi.ProjectStructureInfo
 import com.intellij.python.pyproject.model.spi.PyProjectTomlProject
 import com.intellij.python.pyproject.model.spi.Tool
 import com.intellij.python.sdk.ui.icons.PythonSdkUIIcons
-import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
 import com.jetbrains.python.PyToolUIInfo
 import com.jetbrains.python.ToolId
-import com.jetbrains.python.projectModel.common.getDependenciesFromToml
-import com.jetbrains.python.projectModel.common.getProjectStructure
 import com.jetbrains.python.venvReader.Directory
 import org.apache.tuweni.toml.TomlTable
 import org.jetbrains.annotations.ApiStatus
 
 @ApiStatus.Internal
 val POETRY_TOOL_ID: ToolId = ToolId("poetry")
+
 internal class PoetryTool : Tool {
 
   override val id: ToolId = POETRY_TOOL_ID
@@ -29,18 +26,6 @@ internal class PoetryTool : Tool {
   override suspend fun getProjectName(projectToml: TomlTable): @NlsSafe String? =
     projectToml.getString("tool.poetry.name")
 
-  override suspend fun getProjectStructure(entries: Map<ProjectName, PyProjectTomlProject>, rootIndex: Map<Directory, ProjectName>): ProjectStructureInfo =
-    getProjectStructure(entries, rootIndex) { getDependencies(it.root, it.pyProjectToml) }
+  override suspend fun getProjectStructure(entries: Map<ProjectName, PyProjectTomlProject>, rootIndex: Map<Directory, ProjectName>): ProjectStructureInfo? = null
 
-  @RequiresBackgroundThread
-  private fun getDependencies(rootDir: Directory, projectToml: PyProjectToml): Set<Directory> {
-    val moduleDependenciesSet = getDependenciesFromToml(projectToml)
-    val oldStyleModuleDependencies = projectToml.toml.getTableOrEmpty("tool.poetry.dependencies")
-      .toMap().entries
-      .mapNotNull { (_, depSpec) ->
-        if (depSpec !is TomlTable || depSpec.getBoolean("develop") != true) return@mapNotNull null
-        depSpec.getString("path")?.let { rootDir.resolve(it).normalize() }
-      }
-    return moduleDependenciesSet + oldStyleModuleDependencies.toSet()
-  }
 }

python/src/com/jetbrains/python/projectModel/uv/UvTool.kt

@@ -4,8 +4,10 @@ package com.jetbrains.python.projectModel.uv
 import com.intellij.openapi.diagnostic.fileLogger
 import com.intellij.openapi.util.NlsSafe
 import com.intellij.openapi.util.getPathMatcher
-import com.intellij.python.pyproject.PY_PROJECT_TOML_BUILD_SYSTEM
-import com.intellij.python.pyproject.model.spi.*
+import com.intellij.python.pyproject.model.spi.ProjectName
+import com.intellij.python.pyproject.model.spi.ProjectStructureInfo
+import com.intellij.python.pyproject.model.spi.PyProjectTomlProject
+import com.intellij.python.pyproject.model.spi.Tool
 import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
 import com.jetbrains.python.PyToolUIInfo
 import com.jetbrains.python.ToolId
@@ -23,6 +25,7 @@ import kotlin.io.path.relativeTo
 
 @ApiStatus.Internal
 val UV_TOOL_ID: ToolId = ToolId("uv")
+
 internal class UvTool : Tool {
 
   override val id: ToolId = UV_TOOL_ID