diff --git a/python/pluginCore/resources/META-INF/plugin.xml b/python/pluginCore/resources/META-INF/plugin.xml index 62ad9d0fbbe9..eeb1f6659417 100644 --- a/python/pluginCore/resources/META-INF/plugin.xml +++ b/python/pluginCore/resources/META-INF/plugin.xml @@ -495,6 +495,7 @@ The Python plug-in provides smart editing for Python scripts. The feature set of + diff --git a/python/src/com/jetbrains/python/statistics/PyProjectTomlUsageCollector.kt b/python/src/com/jetbrains/python/statistics/PyProjectTomlUsageCollector.kt new file mode 100644 index 000000000000..08ef8024064d --- /dev/null +++ b/python/src/com/jetbrains/python/statistics/PyProjectTomlUsageCollector.kt @@ -0,0 +1,169 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.jetbrains.python.statistics + +import com.intellij.internal.statistic.beans.MetricEvent +import com.intellij.internal.statistic.eventLog.EventLogGroup +import com.intellij.internal.statistic.eventLog.events.EventFields +import com.intellij.internal.statistic.service.fus.collectors.ProjectUsagesCollector +import com.intellij.openapi.project.Project +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.openapi.vfs.findPsiFile +import com.intellij.psi.PsiFile +import com.intellij.psi.search.FileTypeIndex +import com.intellij.psi.search.GlobalSearchScope +import com.jetbrains.python.packaging.PyRequirementParser +import org.jetbrains.annotations.ApiStatus.Internal +import org.jetbrains.annotations.VisibleForTesting +import org.toml.lang.psi.* + +private val toolsWhiteList = listOf( + "autoflake", + "basedpyright", + "black", + "cibuildwheel", + "cmake", + "codespell", + "comfy", + "conan", + "conda-lock", + "coverage", + "cython", + "flake8", + "flit", + "flit-core", + "hatch", + "hatch-vcs", + "hatchling", + "isort", + "make-env", + "mypy", + "ninja", + "nitpick", + "pdm", + "poe", + "poetry", + "poetry-core", + "pybind11", + "pycln", + "pydantic-mypy", + "pylint", + "pyright", + "pytest", + "pytoniq", + "refurb", + "ruff", + "scikit-build", + "sematic-release", + "setuptools", + "setuptools-rust", + "setuptools-scm", + "vulture", + "wheel", +) + +@Internal +@VisibleForTesting +class PyProjectTomlUsageCollector : ProjectUsagesCollector() { + + private val GROUP = EventLogGroup("python.toml.stats", 1) + private val PYTHON_PYTOML_TOOLS = GROUP.registerEvent( + "python.pyproject.tools", + EventFields.String("name", toolsWhiteList), + ) + + // https://peps.python.org/pep-0518/ + private val PYTHON_BUILD_BACKEND = GROUP.registerEvent( + "python.pyproject.buildsystem", + EventFields.String("name", toolsWhiteList), + ) + + override fun getGroup(): EventLogGroup = GROUP + + override fun requiresReadAccess() = true + + override fun requiresSmartMode() = true + + override fun getMetrics(project: Project): Set { + val tools = mutableSetOf() + val buildSystems = mutableSetOf() + + FileTypeIndex.processFiles( + TomlFileType, + { file: VirtualFile -> + val psiFile = file.findPsiFile(project) + if (file.name == PY_PROJECT_TOML && psiFile != null && psiFile.isValid) { + collectTools(psiFile, tools) + collectBuildBackends(psiFile, buildSystems) + } + + return@processFiles true + }, + GlobalSearchScope.allScope(project)) + + val metrics = mutableSetOf() + tools.forEach { name -> + metrics.add(PYTHON_PYTOML_TOOLS.metric(name)) + } + + buildSystems.forEach { name: String -> + metrics.add(PYTHON_BUILD_BACKEND.metric(name)) + } + + return metrics + } + + companion object { + const val PY_PROJECT_TOML = "pyproject.toml" + const val BUILD_SYSTEM = "build-system" + const val BUILD_REQUIRES = "requires" + const val TOOL_PREFIX = "tool." + + @JvmStatic + fun collectTools(file: PsiFile, tools: MutableSet) { + val collected = file.children.map { element -> + val key = (element as? TomlTable)?.header?.key?.text ?: "" + val name = if (key.startsWith(TOOL_PREFIX)) + key.substringAfter(TOOL_PREFIX, "").substringBefore(".") + else "" + + normalize(name) + }.filter { + it.isNotEmpty() + } + + tools.addAll(collected) + } + + @JvmStatic + fun collectBuildBackends(file: PsiFile, systems: MutableSet) { + val collected = file.children + .filter { element -> + (element as? TomlTable)?.header?.key?.text == BUILD_SYSTEM + }.flatMap { it -> + it.children.filter { line -> + val kv = (line as? TomlKeyValue) + kv?.key?.text == BUILD_REQUIRES && kv.value as? TomlArray != null + }.flatMap { line -> + val array = (line as TomlKeyValue).value as TomlArray + array.elements.mapNotNull { + (it as? TomlLiteral)?.text + } + } + }.mapNotNull { + val requirement = PyRequirementParser.fromLine(normalize(it)) + requirement?.name + } + + systems.addAll(collected) + } + + @JvmStatic + fun normalize(name: String): String { + return name + .replace("_", "-") + .replace(".", "-") + .replace("\"", "") + .lowercase() + } + } +} \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/packaging/PyProjectTomlUsageCollectorTest.kt b/python/testSrc/com/jetbrains/python/packaging/PyProjectTomlUsageCollectorTest.kt new file mode 100644 index 000000000000..66938b6b50e5 --- /dev/null +++ b/python/testSrc/com/jetbrains/python/packaging/PyProjectTomlUsageCollectorTest.kt @@ -0,0 +1,181 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.jetbrains.python.packaging + +import com.jetbrains.python.fixtures.PyTestCase +import com.jetbrains.python.statistics.PyProjectTomlUsageCollector + +class PyProjectTomlStatsTest : PyTestCase() { + fun doTest(text: String, toolNames: Set, backendNames: Set) { + val psiFile = myFixture.configureByText(PyProjectTomlUsageCollector.PY_PROJECT_TOML, text) + + val tools = mutableSetOf() + val backends = mutableSetOf() + + PyProjectTomlUsageCollector.collectTools(psiFile, tools) + PyProjectTomlUsageCollector.collectBuildBackends(psiFile, backends) + + assertSameElements(tools, toolNames) + assertSameElements(backends, backendNames) + } + + fun testEmptyFile() { + val text = """ + """.trimIndent() + + doTest(text, emptySet(), emptySet()) + } + + fun testInvalidFile() { + val text = """ + some abradackadabra + tools.autoflake + backend=autoflake + """.trimIndent() + + doTest(text, emptySet(), emptySet()) + } + + fun testEmptyToml() { + val text = """ + [build-system] + requires = [] + build-backend = [] + """.trimIndent() + + doTest(text, emptySet(), emptySet()) + } + + fun testDeduplicateTools() { + val text = """ + [tool.ruff.isort.sections] + "nextgisweb_env_lib" = ["nextgisweb.env", "nextgisweb.lib"] + "nextgisweb_comp" = ["nextgisweb"] + + [tool.ruff.flake8-tidy-imports] + + [tool.ruff.flake8-tidy-imports.banned-api] + pkg_resources.msg = "Consider importlib.metadata or nextgisweb.imptool.module_path" + """.trimIndent() + + doTest(text, setOf("ruff"), emptySet()) + } + + fun testNormalizeTools() { + val text = """ + [tool.ruff_furr.isort.sections] + "nextgisweb_env_lib" = ["nextgisweb.env", "nextgisweb.lib"] + "nextgisweb_comp" = ["nextgisweb"] + + + """.trimIndent() + + doTest(text, setOf("ruff-furr"), emptySet()) + } + + fun testCollectBuilds() { + val text = """ + [build-system] + requires = ["hatchling", "setuptools >= 61", "flit"] + build-backend = "xxxxyyy.build" + """.trimIndent() + + doTest(text, emptySet(), setOf("hatchling", "setuptools", "flit")) + } + + fun testDeduplicateBuilds() { + val text = """ + [build-system] + requires = ["hatchling", "hatchling >= 61", "flit"] + build-backend = "xxxxyyy.build" + """.trimIndent() + + doTest(text, emptySet(), setOf("hatchling", "flit")) + } + + fun testNormalizeBuilds() { + val text = """ + [build-system] + requires = ["flit_core", "setup.tools >= 61", "flit "] + build-backend = "xxxxyyy.build" + """.trimIndent() + + doTest(text, emptySet(), setOf("flit", "setup-tools", "flit-core")) + } + + fun testPyPiSimpleExample() { + val text = """ + [build-system] + requires = ["hatchling"] + build-backend = "xxxxyyy.build" + + [project] + name = "pypi-simple" + + keywords = [ + "...", + ] + + classifiers = [ + "Development Status :: 5 - Production/Stable", + "...", + ] + + dependencies = [ + "beautifulsoup4 ~= 4.5", + "....", + ] + + [project.optional-dependencies] + tqdm = ["tqdm"] + + [tool.hatch.version] + path = "src/pypi_simple/__init__.py" + + [tool.hatch.envs.default] + python = "3" + + [tool.mypy] + plugins = ["pydantic.mypy"] + + [tool.pydantic-mypy] + init_forbid_extra = true + """.trimIndent() + + doTest(text, setOf("hatch", "mypy", "pydantic-mypy"), setOf("hatchling")) + } + + fun testPyProjectExample() { + val text = """ + [build-system] + build-backend = "setuptools.build_meta" + requires = [ + "setuptools >= 61", + ] + + [project] + name = "wavelet_prosody_toolkit" + + dependencies = [ + "pyyaml", + "...", + ] + + [project.optional-dependencies] + dev = ["pre-commit"] + + [tool.setuptools] + packages = ["wavelet_prosody_toolkit"] + + [tool.black] + line-length = 120 + + [tool.flake8] + max-line-length = 120 + + [tool.basedpyright] + typeCheckingMode = "standard" + """.trimIndent() + + doTest(text, setOf("setuptools", "black", "flake8", "basedpyright"), setOf("setuptools")) + } +} \ No newline at end of file