PY-73001 Log popular tools and build system requirements mentioned in pyproject.toml

GitOrigin-RevId: 8a54afcc73246ff4d2667229345aa1778dc6a2af

python/pluginCore/resources/META-INF/plugin.xml

@@ -495,6 +495,7 @@ The Python plug-in provides smart editing for Python scripts. The feature set of
     <statistics.projectUsagesCollector implementation="com.jetbrains.python.statistics.PyInterpreterUsagesCollector"/>
     <statistics.projectUsagesCollector implementation="com.jetbrains.python.statistics.PyPackageVersionUsagesCollector"/>
     <statistics.projectUsagesCollector implementation="com.jetbrains.python.statistics.PyPackageInEditorUsageCollector"/>
+    <statistics.projectUsagesCollector implementation="com.jetbrains.python.statistics.PyProjectTomlUsageCollector"/>
     <statistics.counterUsagesCollector implementationClass="com.jetbrains.python.namespacePackages.PyNamespacePackagesStatisticsCollector"/>
     <statistics.counterUsagesCollector implementationClass="com.jetbrains.python.codeInsight.codeVision.PyCodeVisionUsageCollector"/>
     <statistics.counterUsagesCollector implementationClass="com.jetbrains.python.newProject.collector.PythonNewProjectWizardCollector"/>

python/src/com/jetbrains/python/statistics/PyProjectTomlUsageCollector.kt

@@ -0,0 +1,169 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.statistics
+
+import com.intellij.internal.statistic.beans.MetricEvent
+import com.intellij.internal.statistic.eventLog.EventLogGroup
+import com.intellij.internal.statistic.eventLog.events.EventFields
+import com.intellij.internal.statistic.service.fus.collectors.ProjectUsagesCollector
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.findPsiFile
+import com.intellij.psi.PsiFile
+import com.intellij.psi.search.FileTypeIndex
+import com.intellij.psi.search.GlobalSearchScope
+import com.jetbrains.python.packaging.PyRequirementParser
+import org.jetbrains.annotations.ApiStatus.Internal
+import org.jetbrains.annotations.VisibleForTesting
+import org.toml.lang.psi.*
+
+private val toolsWhiteList = listOf(
+  "autoflake",
+  "basedpyright",
+  "black",
+  "cibuildwheel",
+  "cmake",
+  "codespell",
+  "comfy",
+  "conan",
+  "conda-lock",
+  "coverage",
+  "cython",
+  "flake8",
+  "flit",
+  "flit-core",
+  "hatch",
+  "hatch-vcs",
+  "hatchling",
+  "isort",
+  "make-env",
+  "mypy",
+  "ninja",
+  "nitpick",
+  "pdm",
+  "poe",
+  "poetry",
+  "poetry-core",
+  "pybind11",
+  "pycln",
+  "pydantic-mypy",
+  "pylint",
+  "pyright",
+  "pytest",
+  "pytoniq",
+  "refurb",
+  "ruff",
+  "scikit-build",
+  "sematic-release",
+  "setuptools",
+  "setuptools-rust",
+  "setuptools-scm",
+  "vulture",
+  "wheel",
+)
+
+@Internal
+@VisibleForTesting
+class PyProjectTomlUsageCollector : ProjectUsagesCollector() {
+
+  private val GROUP = EventLogGroup("python.toml.stats", 1)
+  private val PYTHON_PYTOML_TOOLS = GROUP.registerEvent(
+    "python.pyproject.tools",
+    EventFields.String("name", toolsWhiteList),
+  )
+
+  // https://peps.python.org/pep-0518/
+  private val PYTHON_BUILD_BACKEND = GROUP.registerEvent(
+    "python.pyproject.buildsystem",
+    EventFields.String("name", toolsWhiteList),
+  )
+
+  override fun getGroup(): EventLogGroup = GROUP
+
+  override fun requiresReadAccess() = true
+
+  override fun requiresSmartMode() = true
+
+  override fun getMetrics(project: Project): Set<MetricEvent> {
+    val tools = mutableSetOf<String>()
+    val buildSystems = mutableSetOf<String>()
+
+    FileTypeIndex.processFiles(
+      TomlFileType,
+      { file: VirtualFile ->
+        val psiFile = file.findPsiFile(project)
+        if (file.name == PY_PROJECT_TOML && psiFile != null && psiFile.isValid) {
+          collectTools(psiFile, tools)
+          collectBuildBackends(psiFile, buildSystems)
+        }
+
+        return@processFiles true
+      },
+      GlobalSearchScope.allScope(project))
+
+    val metrics = mutableSetOf<MetricEvent>()
+    tools.forEach { name ->
+      metrics.add(PYTHON_PYTOML_TOOLS.metric(name))
+    }
+
+    buildSystems.forEach { name: String ->
+      metrics.add(PYTHON_BUILD_BACKEND.metric(name))
+    }
+
+    return metrics
+  }
+
+  companion object {
+    const val PY_PROJECT_TOML = "pyproject.toml"
+    const val BUILD_SYSTEM = "build-system"
+    const val BUILD_REQUIRES = "requires"
+    const val TOOL_PREFIX = "tool."
+
+    @JvmStatic
+    fun collectTools(file: PsiFile, tools: MutableSet<String>) {
+      val collected = file.children.map { element ->
+        val key = (element as? TomlTable)?.header?.key?.text ?: ""
+        val name = if (key.startsWith(TOOL_PREFIX))
+          key.substringAfter(TOOL_PREFIX, "").substringBefore(".")
+        else ""
+
+        normalize(name)
+      }.filter {
+        it.isNotEmpty()
+      }
+
+      tools.addAll(collected)
+    }
+
+    @JvmStatic
+    fun collectBuildBackends(file: PsiFile, systems: MutableSet<String>) {
+      val collected = file.children
+        .filter { element ->
+          (element as? TomlTable)?.header?.key?.text == BUILD_SYSTEM
+        }.flatMap { it ->
+          it.children.filter { line ->
+            val kv = (line as? TomlKeyValue)
+            kv?.key?.text == BUILD_REQUIRES && kv.value as? TomlArray != null
+          }.flatMap { line ->
+            val array = (line as TomlKeyValue).value as TomlArray
+            array.elements.mapNotNull {
+              (it as? TomlLiteral)?.text
+            }
+          }
+        }.mapNotNull {
+          val requirement = PyRequirementParser.fromLine(normalize(it))
+          requirement?.name
+        }
+
+      systems.addAll(collected)
+    }
+
+    @JvmStatic
+    fun normalize(name: String): String {
+      return name
+        .replace("_", "-")
+        .replace(".", "-")
+        .replace("\"", "")
+        .lowercase()
+    }
+  }
+}

python/testSrc/com/jetbrains/python/packaging/PyProjectTomlUsageCollectorTest.kt

@@ -0,0 +1,181 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.packaging
+
+import com.jetbrains.python.fixtures.PyTestCase
+import com.jetbrains.python.statistics.PyProjectTomlUsageCollector
+
+class PyProjectTomlStatsTest : PyTestCase() {
+  fun doTest(text: String, toolNames: Set<String>, backendNames: Set<String>) {
+    val psiFile = myFixture.configureByText(PyProjectTomlUsageCollector.PY_PROJECT_TOML, text)
+
+    val tools = mutableSetOf<String>()
+    val backends = mutableSetOf<String>()
+
+    PyProjectTomlUsageCollector.collectTools(psiFile, tools)
+    PyProjectTomlUsageCollector.collectBuildBackends(psiFile, backends)
+
+    assertSameElements(tools, toolNames)
+    assertSameElements(backends, backendNames)
+  }
+
+  fun testEmptyFile() {
+    val text = """
+        """.trimIndent()
+
+    doTest(text, emptySet(), emptySet())
+  }
+
+  fun testInvalidFile() {
+    val text = """
+      some abradackadabra
+      tools.autoflake
+      backend=autoflake
+        """.trimIndent()
+
+    doTest(text, emptySet(), emptySet())
+  }
+
+  fun testEmptyToml() {
+    val text = """
+        [build-system]
+        requires = []
+        build-backend = []
+      """.trimIndent()
+
+    doTest(text, emptySet(), emptySet())
+  }
+
+  fun testDeduplicateTools() {
+    val text = """
+      [tool.ruff.isort.sections]
+      "nextgisweb_env_lib" = ["nextgisweb.env", "nextgisweb.lib"]
+      "nextgisweb_comp" = ["nextgisweb"]
+
+      [tool.ruff.flake8-tidy-imports]
+
+      [tool.ruff.flake8-tidy-imports.banned-api]
+      pkg_resources.msg = "Consider importlib.metadata or nextgisweb.imptool.module_path"
+    """.trimIndent()
+
+    doTest(text, setOf("ruff"), emptySet())
+  }
+
+  fun testNormalizeTools() {
+    val text = """
+      [tool.ruff_furr.isort.sections]
+      "nextgisweb_env_lib" = ["nextgisweb.env", "nextgisweb.lib"]
+      "nextgisweb_comp" = ["nextgisweb"]
+
+     
+    """.trimIndent()
+
+    doTest(text, setOf("ruff-furr"), emptySet())
+  }
+
+  fun testCollectBuilds() {
+    val text = """
+      [build-system]
+      requires = ["hatchling", "setuptools >= 61", "flit"]
+      build-backend = "xxxxyyy.build"
+    """.trimIndent()
+
+    doTest(text, emptySet(), setOf("hatchling", "setuptools", "flit"))
+  }
+
+  fun testDeduplicateBuilds() {
+    val text = """
+      [build-system]
+      requires = ["hatchling", "hatchling >= 61", "flit"]
+      build-backend = "xxxxyyy.build"
+    """.trimIndent()
+
+    doTest(text, emptySet(), setOf("hatchling", "flit"))
+  }
+
+  fun testNormalizeBuilds() {
+    val text = """
+      [build-system]
+      requires = ["flit_core", "setup.tools >= 61", "flit "]
+      build-backend = "xxxxyyy.build"
+    """.trimIndent()
+
+    doTest(text, emptySet(), setOf("flit", "setup-tools", "flit-core"))
+  }
+
+  fun testPyPiSimpleExample() {
+    val text = """
+      [build-system]
+      requires = ["hatchling"]
+      build-backend = "xxxxyyy.build"
+
+      [project]
+      name = "pypi-simple"
+    
+      keywords = [
+          "...",
+      ]
+
+      classifiers = [
+          "Development Status :: 5 - Production/Stable",
+          "...",
+          ]
+
+      dependencies = [
+          "beautifulsoup4 ~= 4.5",
+          "....",
+      ]
+
+      [project.optional-dependencies]
+      tqdm = ["tqdm"]
+
+      [tool.hatch.version]
+      path = "src/pypi_simple/__init__.py"
+
+      [tool.hatch.envs.default]
+      python = "3"
+
+      [tool.mypy]
+      plugins = ["pydantic.mypy"]
+
+      [tool.pydantic-mypy]
+      init_forbid_extra = true
+    """.trimIndent()
+
+    doTest(text, setOf("hatch", "mypy", "pydantic-mypy"), setOf("hatchling"))
+  }
+
+  fun testPyProjectExample() {
+    val text = """
+      [build-system]
+      build-backend = "setuptools.build_meta"
+      requires = [
+        "setuptools >= 61",
+      ]
+
+      [project]
+      name = "wavelet_prosody_toolkit"
+   
+      dependencies = [
+        "pyyaml",
+        "...",
+      ]
+
+      [project.optional-dependencies]
+      dev = ["pre-commit"]
+
+      [tool.setuptools]
+      packages = ["wavelet_prosody_toolkit"]
+
+      [tool.black]
+      line-length = 120
+
+      [tool.flake8]
+      max-line-length = 120
+
+      [tool.basedpyright]
+      typeCheckingMode = "standard"
+    """.trimIndent()
+
+    doTest(text, setOf("setuptools", "black", "flake8", "basedpyright"), setOf("setuptools"))
+  }
+}