diff --git a/python/junit5Tests-framework/src/com/intellij/python/junit5Tests/framework/metaInfo/TestMetaInfoExtension.kt b/python/junit5Tests-framework/src/com/intellij/python/junit5Tests/framework/metaInfo/TestMetaInfoExtension.kt
index 23549d73596d..914a39c90178 100644
--- a/python/junit5Tests-framework/src/com/intellij/python/junit5Tests/framework/metaInfo/TestMetaInfoExtension.kt
+++ b/python/junit5Tests-framework/src/com/intellij/python/junit5Tests/framework/metaInfo/TestMetaInfoExtension.kt
@@ -3,13 +3,8 @@ package com.intellij.python.junit5Tests.framework.metaInfo
import com.intellij.testFramework.PlatformTestUtil
import com.intellij.testFramework.TestDataPath
-import org.junit.jupiter.api.extension.BeforeAllCallback
-import org.junit.jupiter.api.extension.BeforeEachCallback
-import org.junit.jupiter.api.extension.Extension
-import org.junit.jupiter.api.extension.ExtensionContext
+import org.junit.jupiter.api.extension.*
import org.junit.jupiter.api.extension.ExtensionContext.Namespace
-import org.junit.jupiter.api.extension.ParameterContext
-import org.junit.jupiter.api.extension.ParameterResolver
import java.nio.file.Path
import kotlin.io.path.relativeTo
import kotlin.jvm.optionals.getOrNull
@@ -67,11 +62,14 @@ internal class TestMetaInfoExtension : BeforeAllCallback, BeforeEachCallback, Ex
* Calculates a real test data path based on class annotations (resolves $CONTENT_ROOT placeholder).
*/
override fun beforeAll(context: ExtensionContext) {
- val testClassInfo = getAnnotation(context, TestClassInfo::class.java)
- ?: error("Add ${TestClassInfo::class} class level")
-
val testDataPathWithPlaceholders = getAnnotation(context, TestDataPath::class.java)?.value
- val testDataPath = testDataPathWithPlaceholders?.let { testClassInfo.resolvePath(it) }
+
+ val testDataPath = testDataPathWithPlaceholders?.let { pathWithPlaceholders ->
+ val testClassInfo = getAnnotation(context, TestClassInfo::class.java)
+ ?: error("Add ${TestClassInfo::class} class level")
+
+ testClassInfo.resolvePath(pathWithPlaceholders)
+ }
val data = TestClassInfoData(
testDataPath = testDataPath,
diff --git a/python/pluginResources/intellij.python.community.impl.xml b/python/pluginResources/intellij.python.community.impl.xml
index 179dfebe0b42..cb4a95f66616 100644
--- a/python/pluginResources/intellij.python.community.impl.xml
+++ b/python/pluginResources/intellij.python.community.impl.xml
@@ -543,7 +543,6 @@
-
diff --git a/python/python-pyproject/BUILD.bazel b/python/python-pyproject/BUILD.bazel
index 1ca892c6d6fc..e01300ec55e8 100644
--- a/python/python-pyproject/BUILD.bazel
+++ b/python/python-pyproject/BUILD.bazel
@@ -42,6 +42,8 @@ jvm_library(
"//platform/lang-impl",
"//platform/lang-core",
"//platform/backend/observation",
+ "//platform/statistics",
+ "//platform/indexing-api:indexing",
],
exports = ["@lib//:tuweni-toml"]
)
@@ -89,6 +91,9 @@ jvm_library(
"//platform/testFramework:testFramework_test_lib",
"//python/python-venv:community-impl-venv",
"//python/python-venv:community-impl-venv_test_lib",
+ "//platform/statistics",
+ "//platform/statistics:statistics_test_lib",
+ "//platform/indexing-api:indexing",
],
exports = ["@lib//:tuweni-toml"]
)
diff --git a/python/python-pyproject/intellij.python.pyproject.iml b/python/python-pyproject/intellij.python.pyproject.iml
index 8002e43a3ffe..c4afebc6fb80 100644
--- a/python/python-pyproject/intellij.python.pyproject.iml
+++ b/python/python-pyproject/intellij.python.pyproject.iml
@@ -40,5 +40,7 @@
+
+
\ No newline at end of file
diff --git a/python/python-pyproject/resources/intellij.python.pyproject.xml b/python/python-pyproject/resources/intellij.python.pyproject.xml
index aec2fbe263b1..f8a2ea3fe8a5 100644
--- a/python/python-pyproject/resources/intellij.python.pyproject.xml
+++ b/python/python-pyproject/resources/intellij.python.pyproject.xml
@@ -11,6 +11,8 @@
+
+
) {
+ AUTOFLAKE("autoflake"),
+ AUTOIMPORT("autoimport"),
+ BASEDPYRIGHT("basedpyright"),
+ BANDIT("bandit", ".bandit"),
+ BLACK("black", "black.toml"),
+ CIBUILDWHEEL("cibuildwheel"),
+ CHECK_JSONSCHEMA("check-jsonschema"),
+ CODESPELL("codespell", ".codespellrc"),
+ COMFY("comfy"),
+ CONDA_LOCK("conda-lock"),
+ COVERAGE("coverage", ".coveragerc"),
+ CYTHON("cython"),
+ DAGSTER("dagster"),
+ DARKER("darker"),
+ DEPTRY("deptry"),
+ DOCFORMATTER("docformatter"),
+ FLIT("flit", "flit.ini"),
+ FLIT_CORE("flit-core"),
+ FLAKE8("flake8", ".flake8"),
+ GREAT_EXPECTATIONS("great-expectations", "great_expectations.yml"),
+ HATCH("hatch", "hatch.toml", "hatch.lock"),
+ HATCHLING("hatchling"),
+ HATCH_VCS("hatch-vcs"),
+ HYPOTHESIS("hypothesis", ".hypothesis"),
+ ISORT("isort", ".isort.cfg"),
+ MAKE_ENV("make-env"),
+ MKDOCSTRINGS("mkdocstrings"),
+ MYST_PARSER("myst-parser"),
+ MYPY("mypy", "mypy.ini"),
+ NBQA("nbqa"),
+ NINJA("ninja"),
+ NOX("nox", "noxfile.py"),
+ PDOC("pdoc"),
+ PDM("pdm", "pdm.lock"),
+ PIXI("pixi", "pixi.toml", "pixi.lock"),
+ POETRY("poetry", "poetry.lock"),
+ POETRY_CORE("poetry-core"),
+ POE("poe"),
+ PREFECT("prefect", "prefect.yaml"),
+ PYBIND11("pybind11"),
+ PYCLN("pycln"),
+ PYDANTIC_MYPY("pydantic-mypy"),
+ PYRIGHT("pyright", "pyrightconfig.json"),
+ PY_SPY("py-spy"),
+ PYTKDOCS("pytkdocs"),
+ PYTONIQ("pytoniq"),
+ PYUPGRADE("pyupgrade"),
+ REFURB("refurb"),
+ RUFF("ruff", "ruff.toml"),
+ SAFETY("safety"),
+ SCITK_BUILD("scikit-build"),
+ SCITK_BUILD_CORE("scikit-build-core"),
+ SEMATIC_RELEASE("sematic-release"),
+ SETUPTOOLS("setuptools", "setup.py", "setup.cfg"),
+ SETUPTOOLS_RUST("setuptools-rust"),
+ SETUPTOOLS_SCM("setuptools-scm"),
+ SPHINX("sphinx"),
+ TOX("tox", "tox.ini", "tox.toml"),
+ UV("uv", "uv.lock"),
+ VALIDATE_PYPROJECT("validate-pyproject"),
+ VULTURE("vulture"),
+ WHEEL("wheel"),
+ YAPF("yapf", ".style.yapf");
+
+ /**
+ * For backward compatibility using normalized name instead of Enum name for FUS.
+ */
+ val fusName: String get() = normalizedName
+
+ constructor(key: String, vararg markerFileNames: String) : this(key, markerFileNames.toSet())
+
+ companion object {
+ fun findByNormalizedName(normalizedName: String): PythonTool? = entries.find { it.normalizedName == normalizedName }
+ }
+}
+
+internal val TRACKED_DEPENDENCY_GROUPS = listOf(
+ "all", "async", "bench", "build", "ci",
+ "cli", "coverage", "db", "debug", "deploy",
+ "dev", "docs", "examples", "extras", "extras-all",
+ "format", "gpu", "lint", "optional", "profile",
+ "security", "test", "tooling", "typing", "viz"
+)
+internal const val DEPENDENCY_GROUP_OTHER = "other"
+
+private val GROUP = EventLogGroup("python.toml.stats", 2, FUS_RECORDER, "Python Project Statistics")
+
+internal val PYTHON_PYPROJECT_TOOLS = GROUP.registerEvent(
+ "python.pyproject.tools",
+ EventFields.Enum("name", PythonTool::class.java) { it.fusName },
+ "A Python tool defined in the [tool.*] table of pyproject.toml"
+)
+
+// https://peps.python.org/pep-0518/
+internal val PYTHON_PYPROJECT_BUILDSYSTEM = GROUP.registerEvent(
+ "python.pyproject.buildsystem",
+ EventFields.Enum("name", PythonTool::class.java) { it.fusName },
+ "A Python tool defined in build-system.requires of pyproject.toml"
+)
+
+internal val PYTHON_TOOL_MARKERS_DETECTED = GROUP.registerEvent(
+ "python.tool.markers.detected",
+ EventFields.Enum("name", PythonTool::class.java) { it.fusName },
+ "A Python tool detected via tool marker files (e.g., uv.lock, hatch.toml)"
+)
+
+internal val PYTHON_PYPROJECT_DEPENDENCY_GROUP = GROUP.registerEvent(
+ "python.pyproject.dependency.group",
+ EventFields.String("name", TRACKED_DEPENDENCY_GROUPS + DEPENDENCY_GROUP_OTHER),
+ "A dependency group from pyproject.toml"
+)
+
+internal val PYTHON_PYPROJECT_COUNT = GROUP.registerEvent(
+ "python.pyproject.count",
+ EventFields.Int("count"),
+ "Number of pyproject.toml files (python projects) in the workspace"
+)
+
+internal class PythonTomlStatsUsagesCollector : ProjectUsagesCollector() {
+
+ override fun getGroup(): EventLogGroup = GROUP
+
+ override fun requiresReadAccess() = true
+
+ override fun requiresSmartMode() = true
+
+ override fun getMetrics(project: Project): Set {
+ val scope = ProjectScope.getContentScope(project)
+
+ val pyProjectTomlCounter = AtomicInteger(0)
+ val tools = mutableSetOf()
+ val buildSystems = mutableSetOf()
+ val dependencyGroups = mutableSetOf()
+
+ FilenameIndex.processFilesByName(PY_PROJECT_TOML, true, scope) { virtualFile ->
+ virtualFile.findPsiFile(project)?.takeIf { it.isValid }?.let { psiFile ->
+ pyProjectTomlCounter.incrementAndGet()
+ tools.addAll(PyProjectTomlCollector.findDeclaredTools(psiFile))
+ buildSystems.addAll(PyProjectTomlCollector.findBuildSystemRequiresTools(psiFile))
+ dependencyGroups.addAll(PyProjectTomlCollector.findDependencyGroups(psiFile))
+ }
+ true
+ }
+
+ val toolsDetectedByMarkers = PythonTool.entries.filterNot { tool ->
+ FilenameIndex.processFilesByNames(tool.markerFileNames, true, scope, null) { false }
+ }
+
+ val metrics = mutableSetOf()
+
+ metrics.add(PYTHON_PYPROJECT_COUNT.metric(pyProjectTomlCounter.get()))
+ tools.mapTo(metrics) { PYTHON_PYPROJECT_TOOLS.metric(it) }
+ buildSystems.mapTo(metrics) { PYTHON_PYPROJECT_BUILDSYSTEM.metric(it) }
+ dependencyGroups.mapTo(metrics) { PYTHON_PYPROJECT_DEPENDENCY_GROUP.metric(it) }
+ toolsDetectedByMarkers.mapTo(metrics) { PYTHON_TOOL_MARKERS_DETECTED.metric(it) }
+
+ return metrics
+ }
+}
+
+internal object PyProjectTomlCollector {
+ fun findDeclaredTools(file: PsiFile): Set {
+ val declaredTools = file.children.mapNotNullTo(mutableSetOf()) { element ->
+ val toolTomlKey = (element as? TomlTable)?.header?.key?.takeIf {
+ it.segments.firstOrNull()?.text == PY_PROJECT_TOML_TOOL_PREFIX
+ } ?: return@mapNotNullTo null
+
+ val toolNormalizedName = toolTomlKey.segments.getOrNull(1)?.text?.let {
+ PyPackageName.normalizePackageName(it)
+ }
+
+ toolNormalizedName?.let { PythonTool.findByNormalizedName(it) }
+ }
+
+ return declaredTools
+ }
+
+ fun findBuildSystemRequiresTools(file: PsiFile): Set {
+ val buildSystemTables = file.children.mapNotNull { psiElement ->
+ (psiElement as? TomlTable)?.takeIf { it.header.key?.text == PY_PROJECT_TOML_BUILD_SYSTEM }
+ }
+
+ val requiresValues = buildSystemTables.flatMap { tomlTable ->
+ tomlTable.children.mapNotNull { line ->
+ (line as? TomlKeyValue)?.takeIf { kv -> kv.key.text == "requires" }?.value
+ }
+ }
+
+ val literals = requiresValues.flatMap { tomlValue ->
+ (tomlValue as? TomlArray)?.elements?.mapNotNull { (it as? TomlLiteral)?.text } ?: emptyList()
+ }
+
+ val buildTools = literals.mapNotNullTo(mutableSetOf()) {
+ val requirement = PyRequirementParser.fromLine(it.removeSurrounding("\""))
+ requirement?.name?.let { key -> PythonTool.findByNormalizedName(key) }
+ }
+
+ return buildTools
+ }
+
+ fun findDependencyGroups(file: PsiFile): List {
+ val dependencyGroupTables = file.children.mapNotNull { psiElement ->
+ (psiElement as? TomlTable)?.takeIf { it.header.key?.text == PY_PROJECT_TOML_DEPENDENCY_GROUPS }
+ }
+ val dependencyGroups = dependencyGroupTables.flatMap { tomlTable ->
+ tomlTable.children.mapNotNull { (it as? TomlKeyValue)?.key?.text }
+ }.distinct().map { it.takeIf { it in TRACKED_DEPENDENCY_GROUPS } ?: DEPENDENCY_GROUP_OTHER }.sorted()
+
+ return dependencyGroups
+ }
+}
diff --git a/python/testSrc/com/jetbrains/python/packaging/PyProjectTomlUsageCollectorTest.kt b/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PyProjectTomlCollectorTest.kt
similarity index 62%
rename from python/testSrc/com/jetbrains/python/packaging/PyProjectTomlUsageCollectorTest.kt
rename to python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PyProjectTomlCollectorTest.kt
index ed040a2ecb64..ee7f2da44f79 100644
--- a/python/testSrc/com/jetbrains/python/packaging/PyProjectTomlUsageCollectorTest.kt
+++ b/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PyProjectTomlCollectorTest.kt
@@ -1,24 +1,53 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.packaging
+package com.intellij.python.junit5Tests.unit.pyproject
+import com.intellij.openapi.application.EDT
+import com.intellij.openapi.project.Project
+import com.intellij.psi.PsiFileFactory
+import com.intellij.python.junit5Tests.framework.PyDefaultTestApplication
import com.intellij.python.pyproject.PY_PROJECT_TOML
-import com.jetbrains.python.fixtures.PyTestCase
-import com.jetbrains.python.statistics.PyProjectTomlUsageCollector
+import com.intellij.python.pyproject.statistics.DEPENDENCY_GROUP_OTHER
+import com.intellij.python.pyproject.statistics.PyProjectTomlCollector
+import com.intellij.testFramework.UsefulTestCase
+import com.intellij.testFramework.common.timeoutRunBlocking
+import kotlinx.coroutines.Dispatchers
+import org.junit.jupiter.api.Test
+import org.toml.lang.psi.TomlFileType
-class PyProjectTomlStatsTest : PyTestCase() {
- fun doTest(text: String, toolNames: Set, backendNames: Set) {
- val psiFile = myFixture.configureByText(PY_PROJECT_TOML, text)
+@PyDefaultTestApplication
+class PyProjectTomlCollectorTest(val project: Project) {
+ fun doTest(
+ text: String,
+ toolNames: Set = emptySet(),
+ backendNames: Set = emptySet(),
+ dependencyGroupsNames: Set = emptySet(),
+ ) = timeoutRunBlocking(context = Dispatchers.EDT) {
+ val psiFile = PsiFileFactory.getInstance(project).createFileFromText(PY_PROJECT_TOML, TomlFileType, text)
- val tools = mutableSetOf()
- val backends = mutableSetOf()
+ val tools = PyProjectTomlCollector.findDeclaredTools(psiFile).map { it.fusName }
+ val backends = PyProjectTomlCollector.findBuildSystemRequiresTools(psiFile).map { it.fusName }
+ val dependencyGroups = PyProjectTomlCollector.findDependencyGroups(psiFile)
- PyProjectTomlUsageCollector.collectTools(psiFile, tools)
- PyProjectTomlUsageCollector.collectBuildBackends(psiFile, backends)
-
- assertSameElements(tools, toolNames)
- assertSameElements(backends, backendNames)
+ UsefulTestCase.assertSameElements(tools, toolNames)
+ UsefulTestCase.assertSameElements(backends, backendNames)
+ UsefulTestCase.assertSameElements(dependencyGroups, dependencyGroupsNames)
}
+ @Test
+ fun testDependencyGroups() {
+ val text = """
+ [dependency-groups]
+ test = ["pytest<8", "coverage"]
+ typing = ["mypy==1.7.1", "types-requests"]
+
+ [dependency-groups]
+ lint = ["black", "flake8"]
+ typing-test = [{include-group = "typing"}, "pytest<8"]
+ """.trimIndent()
+ doTest(text, dependencyGroupsNames = setOf("test", "typing", "lint", DEPENDENCY_GROUP_OTHER))
+ }
+
+ @Test
fun testEmptyFile() {
val text = """
""".trimIndent()
@@ -26,6 +55,7 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, emptySet(), emptySet())
}
+ @Test
fun testInvalidFile() {
val text = """
some abradackadabra
@@ -36,6 +66,7 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, emptySet(), emptySet())
}
+ @Test
fun testEmptyToml() {
val text = """
[build-system]
@@ -46,6 +77,7 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, emptySet(), emptySet())
}
+ @Test
fun testDeduplicateTools() {
val text = """
[tool.ruff.isort.sections]
@@ -61,18 +93,20 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, setOf("ruff"), emptySet())
}
+ @Test
fun testNormalizeTools() {
val text = """
- [tool.ruff_furr.isort.sections]
+ [tool.py_spy.isort.sections]
"nextgisweb_env_lib" = ["nextgisweb.env", "nextgisweb.lib"]
"nextgisweb_comp" = ["nextgisweb"]
""".trimIndent()
- doTest(text, setOf("ruff-furr"), emptySet())
+ doTest(text, setOf("py-spy"), emptySet())
}
+ @Test
fun testCollectBuilds() {
val text = """
[build-system]
@@ -83,6 +117,7 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, emptySet(), setOf("hatchling", "setuptools", "flit"))
}
+ @Test
fun testDeduplicateBuilds() {
val text = """
[build-system]
@@ -93,6 +128,7 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, emptySet(), setOf("hatchling", "flit"))
}
+ @Test
fun testCommentedBuilds() {
val text = """
[build-system]
@@ -106,16 +142,18 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, emptySet(), setOf("hatchling"))
}
+ @Test
fun testNormalizeBuilds() {
val text = """
[build-system]
- requires = ["flit_core", "setup.tools >= 61", "flit "]
+ requires = ["flit_core", "setuptools >= 61", "flit "]
build-backend = "xxxxyyy.build"
""".trimIndent()
- doTest(text, emptySet(), setOf("flit", "setup-tools", "flit-core"))
+ doTest(text, emptySet(), setOf("flit", "setuptools", "flit-core"))
}
+ @Test
fun testPyPiSimpleExample() {
val text = """
[build-system]
@@ -158,6 +196,7 @@ class PyProjectTomlStatsTest : PyTestCase() {
doTest(text, setOf("hatch", "mypy", "pydantic-mypy"), setOf("hatchling"))
}
+ @Test
fun testPyProjectExample() {
val text = """
[build-system]
diff --git a/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PythonTomlStatsUsagesCollectorTest.kt b/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PythonTomlStatsUsagesCollectorTest.kt
new file mode 100644
index 000000000000..addeea61d3c5
--- /dev/null
+++ b/python/python-pyproject/test/com/intellij/python/junit5Tests/unit/pyproject/PythonTomlStatsUsagesCollectorTest.kt
@@ -0,0 +1,73 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.junit5Tests.unit.pyproject
+
+import com.intellij.openapi.application.EDT
+import com.intellij.openapi.project.Project
+import com.intellij.python.junit5Tests.framework.PyDefaultTestApplication
+import com.intellij.python.junit5Tests.framework.metaInfo.Repository
+import com.intellij.python.junit5Tests.framework.metaInfo.TestClassInfo
+import com.intellij.python.pyproject.statistics.*
+import com.intellij.testFramework.TestDataPath
+import com.intellij.testFramework.common.timeoutRunBlocking
+import kotlinx.coroutines.Dispatchers
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+@PyDefaultTestApplication
+@TestClassInfo(Repository.PY_COMMUNITY)
+@TestDataPath($$"$CONTENT_ROOT/../python-pyproject/testData/statistics/tools")
+class PythonTomlStatsUsagesCollectorTest(val project: Project) {
+ @Test
+ fun testMonorepo() = timeoutRunBlocking(context = Dispatchers.EDT) {
+ val metrics = PythonTomlStatsUsagesCollector().collect(project)
+ val groups = metrics.groupBy({ it.eventId }, { it.data.build() })
+
+ assertEquals(
+ 2,
+ groups[PYTHON_PYPROJECT_COUNT.eventId]!!.single().values.first() as Int
+ ) {
+ "Number of pyproject.toml files does not match the test data"
+ }
+
+ // "viz" was selected as a non-declared group randomly to cover a not-found case
+ val dependencyGroups = groups[PYTHON_PYPROJECT_DEPENDENCY_GROUP.eventId]!!.map { it.values.first() as String }
+ assertEquals(
+ (TRACKED_DEPENDENCY_GROUPS + DEPENDENCY_GROUP_OTHER - listOf("viz")).sorted(),
+ dependencyGroups.sorted(),
+ ) {
+ "Dependency groups logged do not match the test data"
+ }
+
+ // ".bandit" was selected as a non-declared tool randomly to cover a not-found case
+ val detectedToolsViaMarkers = groups[PYTHON_TOOL_MARKERS_DETECTED.eventId]!!.map { it.values.first() as String }
+ assertEquals(
+ (PythonTool.entries.filter { it.markerFileNames.isNotEmpty() && it != PythonTool.BANDIT }).map { it.fusName }.sorted(),
+ detectedToolsViaMarkers.sorted(),
+ ) {
+ "Detected tools via marker files do not match the test data"
+ }
+
+ // tools randomly distributed across [build-system.requires] and [tool.*]
+ val buildSystemTools = setOf(
+ PythonTool.SETUPTOOLS, PythonTool.WHEEL, PythonTool.SETUPTOOLS_SCM, PythonTool.SETUPTOOLS_RUST,
+ PythonTool.FLIT_CORE, PythonTool.HATCHLING, PythonTool.POETRY_CORE, PythonTool.SCITK_BUILD_CORE,
+ PythonTool.CYTHON, PythonTool.PYBIND11, PythonTool.NINJA, PythonTool.SEMATIC_RELEASE, PythonTool.SCITK_BUILD
+ )
+
+ val toolsFromBuildSystem = groups[PYTHON_PYPROJECT_BUILDSYSTEM.eventId]!!.map { it.values.first() as String }
+ assertEquals(
+ (PythonTool.entries.filter { it in buildSystemTools }).map { it.fusName }.sorted(),
+ toolsFromBuildSystem.sorted(),
+ ) {
+ "Detected tools via [build-system.requires] do not match the test data"
+ }
+
+ val toolsFromPyProjectToml = groups[PYTHON_PYPROJECT_TOOLS.eventId]!!.map { it.values.first() as String }
+ assertEquals(
+ (PythonTool.entries.filter { it !in buildSystemTools }).map { it.fusName }.sorted(),
+ toolsFromPyProjectToml.sorted(),
+ ) {
+ "Detected tools [tool.*] via pyproject.toml do not match the test data"
+ }
+ }
+}
\ No newline at end of file
diff --git a/python/python-pyproject/testData/statistics/tools/black.toml b/python/python-pyproject/testData/statistics/tools/black.toml
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/flit.ini b/python/python-pyproject/testData/statistics/tools/flit.ini
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/great_expectations.yml b/python/python-pyproject/testData/statistics/tools/great_expectations.yml
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/hatch.lock b/python/python-pyproject/testData/statistics/tools/hatch.lock
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/mypy.ini b/python/python-pyproject/testData/statistics/tools/mypy.ini
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/noxfile.py b/python/python-pyproject/testData/statistics/tools/noxfile.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/pdm.lock b/python/python-pyproject/testData/statistics/tools/pdm.lock
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/pixi.lock b/python/python-pyproject/testData/statistics/tools/pixi.lock
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/poetry.lock b/python/python-pyproject/testData/statistics/tools/poetry.lock
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/prefect.yaml b/python/python-pyproject/testData/statistics/tools/prefect.yaml
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/pyproject.toml b/python/python-pyproject/testData/statistics/tools/pyproject.toml
new file mode 100644
index 000000000000..c7a42ab91c30
--- /dev/null
+++ b/python/python-pyproject/testData/statistics/tools/pyproject.toml
@@ -0,0 +1,119 @@
+[build-system]
+requires = [
+ "setuptools>=68",
+ "wheel>=0.41",
+ "setuptools_scm>=8",
+ "setuptools_rust>=1.8",
+ "flit-core>=3.9",
+ "hatchling>=1.21",
+ "poetry-core>=1.9",
+ "scikit-build-core>=0.9",
+ "cython>=3.0",
+ "pybind11>=2.11",
+]
+build-backend = "setuptools.build_meta"
+
+[tool]
+
+[tool.bandit]
+
+[tool.black]
+
+[tool.codespell]
+
+[tool.coverage]
+
+[tool.darker]
+
+[tool.deptry]
+
+[tool.docformatter]
+
+[tool.flake8]
+
+[tool.hatch.xxx]
+
+[tool.hatch-vcs]
+
+[tool.hypothesis]
+
+[tool.isort]
+
+[tool.mkdocstrings]
+
+[tool.myst-parser]
+
+[tool.mypy]
+
+[tool.nbqa]
+
+[tool.nox]
+
+[tool.pdm]
+
+[tool.poetry]
+
+[tool.prefect]
+
+[tool.pycln]
+
+[tool.pydantic-mypy]
+
+[tool.pyright]
+
+[tool.pyupgrade]
+
+[tool.refurb]
+
+[tool.ruff]
+
+[tool.safety]
+
+[tool.sphinx]
+
+[tool.tox]
+
+[tool.uv]
+
+[tool.validate-pyproject]
+
+[tool.vulture]
+
+[tool.yapf]
+
+[dependency-groups]
+
+all = ["asyncio", "rich", "httpx", "pytest", "ruff", "mypy"]
+async = ["anyio", "async-timeout", "aiofiles", "httpx"]
+bench = ["pytest-benchmark", "perfplot", "timeit-decorator"]
+build = ["build", "twine", "wheel"]
+ci = ["pytest", "coverage[toml]", "ruff", "mypy", "tox"]
+cli = ["click", "rich", "typer"]
+coverage = ["coverage[toml]", "pytest-cov"]
+db = ["sqlalchemy", "alembic", "psycopg2-binary"]
+debug = ["ipdb", "debugpy", "rich"]
+deploy = ["gunicorn", "uvicorn"]
+dev = ["ipython", "pre-commit", "watchfiles"]
+docs = ["sphinx", "mkdocs", "mkdocstrings[python]", "myst-parser"]
+examples = ["jupyter", "notebook", "matplotlib"]
+extras = ["orjson", "ujson"]
+extras-all = ["orjson", "ujson", "msgpack"]
+format = ["black", "isort", "docformatter"]
+gpu = ["cupy", "torch"]
+lint = ["ruff", "flake8", "pylint"]
+optional = ["importlib-metadata"]
+profile = ["line-profiler", "memory-profiler"]
+security = ["bandit", "safety"]
+test = ["pytest", "hypothesis", "pytest-xdist"]
+tooling = ["nox", "pre-commit", "tox"]
+typing = ["mypy", "types-requests", "types-setuptools"]
+# viz = ["matplotlib", "seaborn", "plotly"]
+
+# ─────────────────────────
+# Additional groups, not listed in allowed (will be logged as "other")
+# ─────────────────────────
+
+logging = ["structlog", "loguru"]
+network = ["requests", "urllib3", "websockets"]
+performance = ["orjson", "ujson", "cython"]
+serialization = ["pydantic", "msgpack", "pyyaml"]
diff --git a/python/python-pyproject/testData/statistics/tools/pyrightconfig.json b/python/python-pyproject/testData/statistics/tools/pyrightconfig.json
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/ruff.toml b/python/python-pyproject/testData/statistics/tools/ruff.toml
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/setup.py b/python/python-pyproject/testData/statistics/tools/setup.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/sub-project/.codespellrc b/python/python-pyproject/testData/statistics/tools/sub-project/.codespellrc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/sub-project/.coveragerc b/python/python-pyproject/testData/statistics/tools/sub-project/.coveragerc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/sub-project/.flake8 b/python/python-pyproject/testData/statistics/tools/sub-project/.flake8
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/sub-project/.hypothesis b/python/python-pyproject/testData/statistics/tools/sub-project/.hypothesis
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/sub-project/.isort.cfg b/python/python-pyproject/testData/statistics/tools/sub-project/.isort.cfg
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/sub-project/.style.yapf b/python/python-pyproject/testData/statistics/tools/sub-project/.style.yapf
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/sub-project/pyproject.toml b/python/python-pyproject/testData/statistics/tools/sub-project/pyproject.toml
new file mode 100644
index 000000000000..0b92d5802ad0
--- /dev/null
+++ b/python/python-pyproject/testData/statistics/tools/sub-project/pyproject.toml
@@ -0,0 +1,37 @@
+[build-system]
+requires = [
+ "scikit-build",
+ "sematic-release",
+ "ninja>=1.11",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.autoflake]
+
+[tool.autoimport]
+
+[tool.basedpyright]
+
+[tool.check-jsonschema]
+
+[tool.cibuildwheel]
+
+[tool.comfy]
+
+[tool.conda-lock]
+
+[tool.dagster]
+
+[tool.flit]
+
+[tool.great-expectations]
+
+[tool.make-env]
+
+[tool.mkdoc]
+[tool.pdoc]
+[tool.pixi]
+[tool.poe]
+[tool.py-spy]
+[tool.pytkdocs]
+[tool.pytoniq]
diff --git a/python/python-pyproject/testData/statistics/tools/tox.ini b/python/python-pyproject/testData/statistics/tools/tox.ini
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/python-pyproject/testData/statistics/tools/uv.lock b/python/python-pyproject/testData/statistics/tools/uv.lock
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/python/src/com/jetbrains/python/statistics/PyProjectTomlUsageCollector.kt b/python/src/com/jetbrains/python/statistics/PyProjectTomlUsageCollector.kt
deleted file mode 100644
index a2ce6c03783f..000000000000
--- a/python/src/com/jetbrains/python/statistics/PyProjectTomlUsageCollector.kt
+++ /dev/null
@@ -1,161 +0,0 @@
-// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.statistics
-
-import com.intellij.internal.statistic.beans.MetricEvent
-import com.intellij.internal.statistic.eventLog.EventLogGroup
-import com.intellij.internal.statistic.eventLog.events.EventFields
-import com.intellij.internal.statistic.service.fus.collectors.ProjectUsagesCollector
-import com.intellij.openapi.project.Project
-import com.intellij.openapi.vfs.VirtualFile
-import com.intellij.openapi.vfs.findPsiFile
-import com.intellij.psi.PsiFile
-import com.intellij.psi.search.FileTypeIndex
-import com.intellij.psi.search.ProjectScope
-import com.intellij.python.pyproject.PY_PROJECT_TOML
-import com.intellij.python.pyproject.PY_PROJECT_TOML_BUILD_SYSTEM
-import com.intellij.python.pyproject.PY_PROJECT_TOML_TOOL_PREFIX
-import com.jetbrains.python.packaging.PyPackageName
-import com.jetbrains.python.packaging.PyRequirementParser
-import org.jetbrains.annotations.ApiStatus.Internal
-import org.jetbrains.annotations.VisibleForTesting
-import org.toml.lang.psi.*
-
-private val toolsWhiteList = listOf(
- "autoflake",
- "basedpyright",
- "black",
- "cibuildwheel",
- "cmake",
- "codespell",
- "comfy",
- "conan",
- "conda-lock",
- "coverage",
- "cython",
- "flake8",
- "flit",
- "flit-core",
- "hatch",
- "hatch-vcs",
- "hatchling",
- "isort",
- "make-env",
- "mypy",
- "ninja",
- "nitpick",
- "pdm",
- "poe",
- "poetry",
- "poetry-core",
- "pybind11",
- "pycln",
- "pydantic-mypy",
- "pylint",
- "pyright",
- "pytest",
- "pytoniq",
- "refurb",
- "ruff",
- "scikit-build",
- "sematic-release",
- "setuptools",
- "setuptools-rust",
- "setuptools-scm",
- "vulture",
- "wheel",
-)
-
-@Internal
-@VisibleForTesting
-class PyProjectTomlUsageCollector : ProjectUsagesCollector() {
-
- private val GROUP = EventLogGroup("python.toml.stats", 1)
- private val PYTHON_PYTOML_TOOLS = GROUP.registerEvent(
- "python.pyproject.tools",
- EventFields.String("name", toolsWhiteList),
- )
-
- // https://peps.python.org/pep-0518/
- private val PYTHON_BUILD_BACKEND = GROUP.registerEvent(
- "python.pyproject.buildsystem",
- EventFields.String("name", toolsWhiteList),
- )
-
- override fun getGroup(): EventLogGroup = GROUP
-
- override fun requiresReadAccess() = true
-
- override fun requiresSmartMode() = true
-
- override fun getMetrics(project: Project): Set {
- val tools = mutableSetOf()
- val buildSystems = mutableSetOf()
-
- FileTypeIndex.processFiles(
- TomlFileType,
- { file: VirtualFile ->
- val psiFile = file.findPsiFile(project)
- if (file.name == PY_PROJECT_TOML && psiFile != null && psiFile.isValid) {
- collectTools(psiFile, tools)
- collectBuildBackends(psiFile, buildSystems)
- }
-
- return@processFiles true
- },
- ProjectScope.getContentScope(project))
-
- val metrics = mutableSetOf()
- tools.forEach { name ->
- metrics.add(PYTHON_PYTOML_TOOLS.metric(name))
- }
-
- buildSystems.forEach { name: String ->
- metrics.add(PYTHON_BUILD_BACKEND.metric(name))
- }
-
- return metrics
- }
-
- companion object {
- const val BUILD_REQUIRES = "requires"
-
- @JvmStatic
- fun collectTools(file: PsiFile, tools: MutableSet) {
- val collected = file.children.map { element ->
- val key = (element as? TomlTable)?.header?.key?.text ?: ""
- val name = if (key.startsWith(PY_PROJECT_TOML_TOOL_PREFIX))
- key.substringAfter(PY_PROJECT_TOML_TOOL_PREFIX, "").substringBefore(".")
- else ""
-
- PyPackageName.normalizePackageName(name)
- }.filter {
- it.isNotEmpty()
- }
-
- tools.addAll(collected)
- }
-
- @JvmStatic
- fun collectBuildBackends(file: PsiFile, systems: MutableSet) {
- val collected = file.children
- .filter { element ->
- (element as? TomlTable)?.header?.key?.text == PY_PROJECT_TOML_BUILD_SYSTEM
- }.flatMap { it ->
- it.children.filter { line ->
- val kv = (line as? TomlKeyValue)
- kv?.key?.text == BUILD_REQUIRES && kv.value as? TomlArray != null
- }.flatMap { line ->
- val array = (line as TomlKeyValue).value as TomlArray
- array.elements.mapNotNull {
- (it as? TomlLiteral)?.text
- }
- }
- }.mapNotNull {
- val requirement = PyRequirementParser.fromLine(PyPackageName.normalizePackageName(it))
- requirement?.name
- }
-
- systems.addAll(collected)
- }
- }
-}
\ No newline at end of file