From 76a5a5e7fd1e8ecebafbbe325169116f531d5fa6 Mon Sep 17 00:00:00 2001 From: Anton Efimchuk Date: Wed, 10 Jun 2026 11:12:26 +0000 Subject: [PATCH] IJPL-222854 support notebook documents in Python LSP Route Jupyter notebooks through the LSP notebook protocol so Python language servers work with cell subdocuments instead of the whole .ipynb file. The adapter maps host positions and ranges to cell URIs, sends notebook open/change/close/save notifications, and keeps python-lsp-core decoupled from Jupyter through a notebook language resolver EP. Register the Jupyter resolver from the Python/Jupyter plugin and let Python LSP descriptors opt into notebook-backed files. Skip Ty pull diagnostics for notebooks until Ty supports notebook diagnostics, while keeping diagnostics enabled for regular Python files. Add Ty and Pyrefly notebook LSP environment coverage for protocol notifications, code insight requests, range mapping, type inference, and the Ty diagnostics regression. Fix the Python test-env UV bootstrap used by that coverage by handling flat Windows zip archives separately from Unix archives with a root directory. GitOrigin-RevId: f958d48142cc376473a2be47b618e00366643f18 --- .../visualization/testFramework/BUILD.bazel | 4 +++ ....notebooks.visualization.testFramework.iml | 1 + ....notebooks.visualization.testFramework.xml | 26 +++++++++++++++++++ .../test/env/core/PyEnvironmentProvider.kt | 13 +++++----- .../intellij/python/test/env/core/install.kt | 8 +++--- .../junit5Tests/framework/env/pySdkFixture.kt | 12 ++++++--- .../env/junit5/UvExecutableProviderTest.kt | 20 ++++++++++++++ .../test/env/uv/UvExecutableProvider.kt | 21 +++++++++++---- 8 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 notebooks/visualization/testFramework/resources/intellij.notebooks.visualization.testFramework.xml create mode 100644 python/python-test-env/junit5/test/com/intellij/python/test/env/junit5/UvExecutableProviderTest.kt diff --git a/notebooks/visualization/testFramework/BUILD.bazel b/notebooks/visualization/testFramework/BUILD.bazel index 8c56ece173c2..82ae6666deb5 100644 --- a/notebooks/visualization/testFramework/BUILD.bazel +++ b/notebooks/visualization/testFramework/BUILD.bazel @@ -23,6 +23,10 @@ jvm_library( ), kotlinc_opts = ":custom_testFramework", module_name = "intellij.notebooks.visualization.testFramework", + resource_strip_prefix = "resources", + resources = glob( + ["resources/**/*"], + ), visibility = ["//visibility:public"], deps = [ # do not sort, diff --git a/notebooks/visualization/testFramework/intellij.notebooks.visualization.testFramework.iml b/notebooks/visualization/testFramework/intellij.notebooks.visualization.testFramework.iml index a861688c09e2..68f67159d85e 100644 --- a/notebooks/visualization/testFramework/intellij.notebooks.visualization.testFramework.iml +++ b/notebooks/visualization/testFramework/intellij.notebooks.visualization.testFramework.iml @@ -24,6 +24,7 @@ + diff --git a/notebooks/visualization/testFramework/resources/intellij.notebooks.visualization.testFramework.xml b/notebooks/visualization/testFramework/resources/intellij.notebooks.visualization.testFramework.xml new file mode 100644 index 000000000000..dc223f5a8f2d --- /dev/null +++ b/notebooks/visualization/testFramework/resources/intellij.notebooks.visualization.testFramework.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/python/python-test-env/core/src/com/intellij/python/test/env/core/PyEnvironmentProvider.kt b/python/python-test-env/core/src/com/intellij/python/test/env/core/PyEnvironmentProvider.kt index 15c94c5707ec..5d4152cb41a4 100644 --- a/python/python-test-env/core/src/com/intellij/python/test/env/core/PyEnvironmentProvider.kt +++ b/python/python-test-env/core/src/com/intellij/python/test/env/core/PyEnvironmentProvider.kt @@ -97,16 +97,15 @@ abstract class PyEnvironmentProvider>( abstract suspend fun setupEnvironment(context: Context, spec: S): PyEnvironment /** - * Extract archive using Decompressor framework. + * Extract archive using the root directory used by Python archives. * Supports tar.gz, .tgz, and .zip formats. - * + * * @param archiveFile Path to the archive file * @param targetDir Directory where archive contents will be extracted - * @param prefixToStrip Path prefix to strip during extraction (e.g., "uv-x86_64-unknown-linux-gnu" for UV archives) * @throws IllegalArgumentException if archive format is not supported */ - protected fun unpackArchive(archiveFile: Path, targetDir: Path, prefixToStrip: String? = null) { - com.intellij.python.test.env.core.unpackArchive(archiveFile, targetDir, prefixToStrip) + protected fun unpackArchive(archiveFile: Path, targetDir: Path) { + unpackArchive(archiveFile, targetDir, prefixToStrip = "python") } /** @@ -119,13 +118,13 @@ abstract class PyEnvironmentProvider>( * Can be used to create nested or dependent environments. */ val factory: PyEnvironmentFactory - + /** * Working directory where environments are created. * Each environment typically gets its own subdirectory. */ val workingDir: Path - + /** * Cache directory for storing downloaded artifacts. * Used to avoid re-downloading Python distributions and other resources. diff --git a/python/python-test-env/core/src/com/intellij/python/test/env/core/install.kt b/python/python-test-env/core/src/com/intellij/python/test/env/core/install.kt index eba64c6625bc..026e9a3c6fa8 100644 --- a/python/python-test-env/core/src/com/intellij/python/test/env/core/install.kt +++ b/python/python-test-env/core/src/com/intellij/python/test/env/core/install.kt @@ -2,6 +2,7 @@ package com.intellij.python.test.env.core import com.intellij.openapi.diagnostic.Logger import com.intellij.util.io.Decompressor +import com.intellij.util.system.LowLevelLocalMachineAccess import com.intellij.util.system.OS import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -91,6 +92,7 @@ suspend fun installPipPackages( executeProcess(command, logger, "pip") } +@OptIn(LowLevelLocalMachineAccess::class) fun markExecutable(logger: Logger, executable: Path) { if (OS.CURRENT != OS.Windows) { logger.info("Setting executable permissions") @@ -111,16 +113,15 @@ fun markExecutable(logger: Logger, executable: Path) { * * @param archiveFile Path to the archive file * @param targetDir Directory where archive contents will be extracted - * @param prefixToStrip Path prefix to strip during extraction (e.g., "uv-x86_64-unknown-linux-gnu" for UV archives) + * @param prefixToStrip Path prefix to strip during extraction. If null, archive paths are extracted as-is. * @throws IllegalStateException if archive format is not supported */ @ApiStatus.Internal -fun unpackArchive(archiveFile: Path, targetDir: Path, prefixToStrip: String? = null) { +fun unpackArchive(archiveFile: Path, targetDir: Path, prefixToStrip: String?) { val fileName = archiveFile.fileName.toString() when { fileName.endsWith(".tar.gz", ignoreCase = true) || fileName.endsWith(".tgz", ignoreCase = true) -> { val decompressor = Decompressor.Tar(archiveFile) - .removePrefixPath("python") if (prefixToStrip != null) { decompressor.removePrefixPath(prefixToStrip) } @@ -128,7 +129,6 @@ fun unpackArchive(archiveFile: Path, targetDir: Path, prefixToStrip: String? = n } fileName.endsWith(".zip", ignoreCase = true) -> { val decompressor = Decompressor.Zip(archiveFile).withZipExtensions() - .removePrefixPath("python") if (prefixToStrip != null) { decompressor.removePrefixPath(prefixToStrip) } diff --git a/python/python-test-env/junit5/src/com/intellij/python/junit5Tests/framework/env/pySdkFixture.kt b/python/python-test-env/junit5/src/com/intellij/python/junit5Tests/framework/env/pySdkFixture.kt index 13aa3624e992..a26baa4c8c4e 100644 --- a/python/python-test-env/junit5/src/com/intellij/python/junit5Tests/framework/env/pySdkFixture.kt +++ b/python/python-test-env/junit5/src/com/intellij/python/junit5Tests/framework/env/pySdkFixture.kt @@ -6,9 +6,9 @@ import com.intellij.openapi.application.writeAction import com.intellij.openapi.projectRoots.ProjectJdkTable import com.intellij.openapi.projectRoots.Sdk import com.intellij.python.test.env.common.PredefinedPyEnvironments -import com.intellij.python.test.env.common.createEnvironment import com.intellij.python.test.env.core.PyEnvironment import com.intellij.python.test.env.core.PyEnvironmentFactory +import com.intellij.python.test.env.core.PyEnvironmentSpec import com.intellij.python.test.env.junit5.RunOnEnvironmentsExtension import com.intellij.python.test.env.junit5.getOrCreatePyEnvironmentFactory import com.intellij.testFramework.junit5.fixture.TestFixture @@ -31,16 +31,20 @@ fun pySdkFixture(): TestFixture> = testFixture { conte */ fun pySdkFixture( env: PredefinedPyEnvironments, +): TestFixture> = pySdkFixture(env.spec) + +fun pySdkFixture( + envSpec: PyEnvironmentSpec<*>, ): TestFixture> = testFixture { context -> val factory = getOrCreatePyEnvironmentFactory(context.extensionContext) - initializedPySdkFixture(factory, env) + initializedPySdkFixture(factory, envSpec) } private suspend fun TestFixtureInitializer.R>.initializedPySdkFixture( factory: PyEnvironmentFactory, - env: PredefinedPyEnvironments, + envSpec: PyEnvironmentSpec<*>, ): TestFixtureInitializer.InitializedTestFixture> { - val env = factory.createEnvironment(env) + val env = factory.createEnvironment(envSpec) return initializedTestFixture(env) } diff --git a/python/python-test-env/junit5/test/com/intellij/python/test/env/junit5/UvExecutableProviderTest.kt b/python/python-test-env/junit5/test/com/intellij/python/test/env/junit5/UvExecutableProviderTest.kt new file mode 100644 index 000000000000..0396159be70a --- /dev/null +++ b/python/python-test-env/junit5/test/com/intellij/python/test/env/junit5/UvExecutableProviderTest.kt @@ -0,0 +1,20 @@ +// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.test.env.junit5 + +import com.intellij.python.test.env.uv.getUvArchivePrefixToStrip +import com.intellij.util.system.OS +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +internal class UvExecutableProviderTest { + @Test + fun `Windows uv archive is extracted without prefix stripping`() { + assertNull(getUvArchivePrefixToStrip(OS.Windows, "uv-x86_64-pc-windows-msvc.zip")) + } + + @Test + fun `Unix uv archive is extracted from its root directory`() { + assertEquals("uv-x86_64-unknown-linux-gnu", getUvArchivePrefixToStrip(OS.Linux, "uv-x86_64-unknown-linux-gnu.tar.gz")) + } +} diff --git a/python/python-test-env/uv/src/com/intellij/python/test/env/uv/UvExecutableProvider.kt b/python/python-test-env/uv/src/com/intellij/python/test/env/uv/UvExecutableProvider.kt index 3cb44431838d..576732f34dc8 100644 --- a/python/python-test-env/uv/src/com/intellij/python/test/env/uv/UvExecutableProvider.kt +++ b/python/python-test-env/uv/src/com/intellij/python/test/env/uv/UvExecutableProvider.kt @@ -7,6 +7,7 @@ import com.intellij.python.test.env.core.extractIfNecessary import com.intellij.python.test.env.core.markExecutable import com.intellij.python.test.env.core.unpackArchive import com.intellij.util.system.CpuArch +import com.intellij.util.system.LowLevelLocalMachineAccess import com.intellij.util.system.OS import org.jetbrains.annotations.ApiStatus import java.nio.file.Path @@ -22,12 +23,14 @@ private val LOG = Logger.getInstance("com.intellij.python.test.env.uv") * @return Path to the UV executable */ @ApiStatus.Internal +@OptIn(LowLevelLocalMachineAccess::class) suspend fun getOrDownloadUvExecutable(uvVersion: String): Path { val logger = LOG val downloadUrl = getUvDownloadUrl(uvVersion) val archiveFileName = downloadUrl.substringAfterLast('/') val archiveBaseName = archiveFileName.substringBeforeLast(".tar.gz").substringBeforeLast(".zip") + val prefixToStrip = getUvArchivePrefixToStrip(OS.CURRENT, archiveFileName) val uvDir = PyEnvDownloadCache.cacheDirectory().resolve("uv").resolve(archiveBaseName) val uvExecutable = uvDir.resolve(if (OS.CURRENT == OS.Windows) "uv.exe" else "uv") @@ -35,14 +38,20 @@ suspend fun getOrDownloadUvExecutable(uvVersion: String): Path { logger.info("Downloading UV archive: $archiveFileName") val cachedArchive = PyEnvDownloadCache.getOrDownload(downloadUrl, archiveFileName) logger.info("UV archive downloaded: $cachedArchive") - logger.info("Extracting UV binary (stripping prefix: $archiveBaseName)") - unpackArchive(cachedArchive, target, prefixToStrip = archiveBaseName) + logger.info("Extracting UV binary (stripping prefix: $prefixToStrip)") + unpackArchive(cachedArchive, target, prefixToStrip = prefixToStrip) markExecutable(logger, uvExecutable) } return uvExecutable } +@ApiStatus.Internal +fun getUvArchivePrefixToStrip(os: OS, archiveFileName: String): String? { + return if (os == OS.Windows) null else archiveFileName.substringBeforeLast(".tar.gz").substringBeforeLast(".zip") +} + +@OptIn(LowLevelLocalMachineAccess::class) private fun getUvDownloadUrl(version: String): String { val arch = when { CpuArch.isArm64() -> "aarch64" @@ -50,12 +59,14 @@ private fun getUvDownloadUrl(version: String): String { else -> "x86_64" } - val platform = when (OS.CURRENT) { + val os = OS.CURRENT + val platform = when (os) { OS.Windows -> "$arch-pc-windows-msvc" OS.macOS -> "$arch-apple-darwin" OS.Linux -> "$arch-unknown-linux-gnu" - else -> throw IllegalStateException("Unsupported OS: ${OS.CURRENT}") + else -> throw IllegalStateException("Unsupported OS: $os") } + val extension = if (os == OS.Windows) "zip" else "tar.gz" - return "$UV_RELEASES_URL/$version/uv-$platform.tar.gz" + return "$UV_RELEASES_URL/$version/uv-$platform.$extension" }