diff --git a/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ThrowingSupplier.java b/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ThrowingSupplier.java index 6aef95e9c625..c39825c459f9 100644 --- a/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ThrowingSupplier.java +++ b/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ThrowingSupplier.java @@ -2,6 +2,6 @@ package org.jetbrains.idea.maven.aether; @FunctionalInterface -interface ThrowingSupplier { +public interface ThrowingSupplier { R get() throws Exception; } diff --git a/java/idea-ui/intellij.java.ui.iml b/java/idea-ui/intellij.java.ui.iml index 8a585e3dd542..ee655c15a4fc 100644 --- a/java/idea-ui/intellij.java.ui.iml +++ b/java/idea-ui/intellij.java.ui.iml @@ -38,5 +38,6 @@ + \ No newline at end of file diff --git a/java/idea-ui/intellij.java.ui.tests.iml b/java/idea-ui/intellij.java.ui.tests.iml index 8ab1d4c39de6..21fa0bc41392 100644 --- a/java/idea-ui/intellij.java.ui.tests.iml +++ b/java/idea-ui/intellij.java.ui.tests.iml @@ -23,5 +23,7 @@ + + \ No newline at end of file diff --git a/java/idea-ui/src/com/intellij/jarRepository/JarHttpDownloader.kt b/java/idea-ui/src/com/intellij/jarRepository/JarHttpDownloader.kt new file mode 100644 index 000000000000..dc816348b21f --- /dev/null +++ b/java/idea-ui/src/com/intellij/jarRepository/JarHttpDownloader.kt @@ -0,0 +1,273 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.jarRepository + +import com.google.common.net.HttpHeaders +import com.intellij.jarRepository.JarRepositoryAuthenticationDataProvider.AuthenticationData +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.util.io.FileUtil +import com.intellij.util.containers.ContainerUtil +import com.intellij.util.io.HttpRequests +import com.intellij.util.io.sha256Hex +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.VisibleForTesting +import org.jetbrains.idea.maven.aether.Retry +import org.jetbrains.idea.maven.aether.ThrowingSupplier +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.util.* +import kotlin.io.path.* + +@ApiStatus.Internal +object JarHttpDownloader { + private val LOG = Logger.getInstance(JarHttpDownloader::class.java) + + @VisibleForTesting + @Volatile + @JvmField + var forceHttps: Boolean = true + + suspend fun downloadLibraryFilesAsync( + relativePaths: List, + localRepository: Path, + remoteRepositories: List, + retry: Retry, + downloadDispatcher: CoroutineDispatcher, + ): List { + if (LOG.isTraceEnabled) { + LOG.trace("Downloading roots $relativePaths, localRepository=$localRepository, remoteRepositories=$remoteRepositories") + } + + // make a maximum effort to download all roots, + // so postpone exception throwing until we try to download all files + + val errors = ContainerUtil.createConcurrentList() + val downloadedFiles = coroutineScope { + relativePaths.map { relativePath -> + async(downloadDispatcher) { + try { + downloadArtifact( + artifactPath = relativePath, + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = retry, + ) + } + catch (t: Throwable) { + errors.add(t) + return@async null + } + } + }.awaitAll().filterNotNull() + } + + if (!errors.isEmpty()) { + val first = errors.first() + val exception = IllegalStateException("Failed to download ${errors.size} artifact(s): (first exception) ${first.message}", first) + errors.drop(1).forEach { exception.addSuppressed(it) } + + if (LOG.isTraceEnabled) { + LOG.trace(exception.stackTraceToString()) + } + + throw exception + } + + return downloadedFiles + } + + fun downloadArtifact( + artifactPath: RelativePathToDownload, + localRepository: Path, + remoteRepositories: List, + retry: Retry, + ): Path { + val targetFile = localRepository.resolve(artifactPath.relativePath) + if (targetFile.exists()) { + if (artifactPath.expectedSha256 != null) { + val actualSha256 = sha256Hex(targetFile) + check(actualSha256 == artifactPath.expectedSha256) { + "Wrong file checksum on disk for '$targetFile': expected checksum ${artifactPath.expectedSha256}, " + + "but got $actualSha256 (fileSize: ${Files.size(targetFile)})" + } + } + + return targetFile + } + + val systemIndependentNormalizedRelativePath = FileUtil.toSystemIndependentName(artifactPath.relativePath.pathString) + val remoteRepositoriesAndUrl = remoteRepositories.map { remoteRepository -> + (remoteRepository.url.trimEnd('/') + "/" + systemIndependentNormalizedRelativePath) to remoteRepository + } + + val authExceptions = mutableListOf>() + for ((url, remoteRepository) in remoteRepositoriesAndUrl) { + try { + val headers = if (remoteRepository.authenticationData == null) { + emptyMap() + } + else { + val credentials = "${remoteRepository.authenticationData.userName}:${remoteRepository.authenticationData.password}" + val authHeader = "Basic " + Base64.getEncoder().encodeToString(credentials.toByteArray()) + mapOf(HttpHeaders.AUTHORIZATION to authHeader) + } + + downloadFile( + url = url, + targetFile = targetFile, + headers = headers, + expectedSha256 = artifactPath.expectedSha256, + retry = retry, + ) + + // success + return targetFile + } + catch (e: HttpRequests.HttpStatusException) { + if (e.statusCode == 404) { + // continue with the next repository + continue + } + + if (e.statusCode == 401) { + // continue, but remember the error + // if all repositories return 404 and some 401, we must throw unauthenticated exception + authExceptions.add(url to e) + continue + } + + throw e + } + } + + if (authExceptions.isNotEmpty()) { + val exception = IllegalStateException( + "Artifact '$systemIndependentNormalizedRelativePath' was not found in remote repositories, " + + "some of them returned 401 Unauthorized: ${authExceptions.map { it.first }}") + authExceptions.forEach { exception.addSuppressed(it.second) } + throw exception + } + + error("Artifact '$systemIndependentNormalizedRelativePath' was not found in remote repositories: ${remoteRepositoriesAndUrl.map { it.first }}") + } + + fun downloadFile( + url: String, + targetFile: Path, + retry: Retry, + headers: Map = emptyMap(), + expectedSha256: String? = null, + ) { + if (forceHttps) { + check(url.startsWith("https://")) { + "Url must have https protocol: $url" + } + } + + LOG.trace("Starting downloading '$url' to '$targetFile', headers=${headers.keys.sorted()}, expectedSha256=$expectedSha256") + + val targetDirectory = targetFile.parent!! + + targetDirectory.createDirectories() + val tempFile = Files.createTempFile(targetDirectory, "." + targetFile.name, ".tmp") + tempFile.deleteIfExists() + try { + var lastFileSize: Long? = null + + val exception = retry.retry(ThrowingSupplier { + try { + HttpRequests.request(url) + .tuner { tuner -> + headers.forEach { (name, value) -> tuner.addRequestProperty(name, value) } + } + .productNameAsUserAgent() + .connect { processor -> + processor.saveToFile(tempFile, null) + + val contentLength = processor.connection.getHeaderFieldLong(HttpHeaders.CONTENT_LENGTH, -1) + check(contentLength > 0) { "Header '${HttpHeaders.CONTENT_LENGTH}' is missing or zero for $url" } + + val contentEncoding = headers[HttpHeaders.CONTENT_ENCODING] + check(contentEncoding == null || contentEncoding == "identity") { + "Unsupported encoding '$contentEncoding' for $url. Only 'identity' encoding is supported" + } + + val fileSize = Files.size(tempFile) + check(fileSize == contentLength) { + "Wrong file length after downloading uri '$url' to '$tempFile': expected length $contentLength " + + "from ${HttpHeaders.CONTENT_ENCODING} header, but got $fileSize on disk" + } + + lastFileSize = fileSize + + if (expectedSha256 != null) { + val actualSha256 = sha256Hex(tempFile) + if (actualSha256 != expectedSha256) { + throw BadChecksumException( + "Wrong file checksum after downloading '$url' to '$tempFile': expected checksum $expectedSha256, " + + "but got $actualSha256 (fileSize: $fileSize)" + ) + } + } + + Files.move(tempFile, targetFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING) + } + + null + } + catch (e: HttpRequests.HttpStatusException) { + if (e.statusCode >= 200 && e.statusCode < 500) { + // those codes should not be retried, changes in answer are unexpected by HTTP standard + e + } + else { + // continue retrying + throw e + } + } + catch (e: BadChecksumException) { + // no retry on bad checksum + e + } + }, LOG) + + if (exception != null) { + if (LOG.isTraceEnabled) { + LOG.trace("Downloading of '$url' to '$targetFile' failed (headers=${headers.keys.sorted()}: ${exception.stackTraceToString()}") + } + + throw exception + } + else { + if (LOG.isTraceEnabled) { + LOG.trace("Downloaded file from '$url' to '$targetFile', size=${lastFileSize}, headers=${headers.keys.sorted()}, expectedSha256=$expectedSha256") + } + + return + } + } + finally { + tempFile.deleteIfExists() + } + } + + data class RemoteRepository(val url: String, val authenticationData: AuthenticationData?) + + data class RelativePathToDownload(val relativePath: Path, val expectedSha256: String?) { + init { + require(!relativePath.isAbsolute) { + "Path $relativePath should be relative" + } + + require(relativePath.normalize().pathString == relativePath.pathString) { + "Path $relativePath should be normalized" + } + } + } + + class BadChecksumException(message: String) : RuntimeException(message) +} diff --git a/java/idea-ui/src/com/intellij/jarRepository/JarHttpDownloaderJps.kt b/java/idea-ui/src/com/intellij/jarRepository/JarHttpDownloaderJps.kt new file mode 100644 index 000000000000..988c5bfa57ff --- /dev/null +++ b/java/idea-ui/src/com/intellij/jarRepository/JarHttpDownloaderJps.kt @@ -0,0 +1,250 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.jarRepository + +import com.intellij.jarRepository.JarHttpDownloader.RelativePathToDownload +import com.intellij.jarRepository.JarHttpDownloader.RemoteRepository +import com.intellij.jarRepository.JarRepositoryAuthenticationDataProvider.AuthenticationData +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.components.PathMacroManager +import com.intellij.openapi.components.Service +import com.intellij.openapi.components.service +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.project.Project +import com.intellij.openapi.roots.AnnotationOrderRootType +import com.intellij.openapi.roots.OrderRootType +import com.intellij.openapi.roots.impl.libraries.LibraryEx +import com.intellij.openapi.util.io.FileUtil +import com.intellij.openapi.util.registry.Registry +import com.intellij.openapi.vfs.LocalFileSystem +import com.intellij.openapi.vfs.VfsUtil +import com.intellij.openapi.vfs.newvfs.VfsImplUtil +import kotlinx.coroutines.* +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.TestOnly +import org.jetbrains.concurrency.AsyncPromise +import org.jetbrains.concurrency.Promise +import org.jetbrains.idea.maven.aether.RetryProvider +import org.jetbrains.idea.maven.utils.library.RepositoryLibraryProperties +import org.jetbrains.jps.model.serialization.JpsMavenSettings +import java.nio.file.Path +import java.util.concurrent.ConcurrentLinkedDeque +import kotlin.io.path.exists +import kotlin.io.path.pathString +import kotlin.io.path.relativeTo + +/** + * Alternative repository libraries downloader, independent of maven code + * Downloads only files listed in + */ +@ApiStatus.Internal +@Service(Service.Level.PROJECT) +class JarHttpDownloaderJps(val project: Project, val coroutineScope: CoroutineScope) { + companion object { + private val LOG = Logger.getInstance(JarHttpDownloaderJps::class.java) + + @JvmStatic + fun enabled(): Boolean = Registry.`is`("jar.http.downloader.enabled") + + @JvmStatic + fun getInstance(project: Project): JarHttpDownloaderJps = project.service() + + private fun collectRelativePathsForJarHttpDownloaderOrLog(library: LibraryEx): CollectResult { + if (library.getKind() != RepositoryLibraryType.REPOSITORY_LIBRARY_KIND) { + return CollectResult.Failure("Library '${library.name}' is not a repository library") + } + + val libraryProperties = library.properties as? RepositoryLibraryProperties + if (libraryProperties == null) { + return CollectResult.Failure("Library '${library.name}' has no repository library properties") + } + + if (!isLibraryHasFixedVersion(libraryProperties)) { + return CollectResult.Failure("Library '${library.name}' does not have fixed version (version=${libraryProperties.version})") + } + + val verification = libraryProperties.artifactsVerification.associate { it.url to it } + + // All paths in .idea/libraries are typically relative to $MAVEN_REPOSITORY$ + // that's the case we want to handle + + // a collection of edge cases where the actual maven local repository root can be + // 1. canonical path (symlinks can be resolved or not resolved) + // 2. in tests, JarRepositoryManager.localRepositoryPath can be overridden + val possibleMavenLocalRepositoryRoots = listOfNotNull( + // could be overridden, like in tests + JarRepositoryManager.getLocalRepositoryPath().path, + + // always returns a canonical path (symlinks resolved), so can be anything even if it was not overridden + PathMacroManager.getInstance(ApplicationManager.getApplication()).expandPath(JarRepositoryManager.MAVEN_REPOSITORY_MACRO), + + // in some cases, we may receive a non-canonical path (without symlinks resolved) + JpsMavenSettings.getMavenRepositoryPath(), + ).map { Path.of(FileUtil.toSystemDependentName(it)).normalize() } + + val files = OrderRootType.getAllTypes().flatMap { rootType -> library.getUrls(rootType).map { rootType to it } } + .mapNotNull { (rootType, url) -> + val urlToPath = VfsUtil.urlToPath(url) + if (urlToPath == url) { + return CollectResult.Failure("Library '${library.name}': root '$url' could not be converted to path") + } + + val independentPath = urlToPath.removeSuffix("!/") + + val path = Path.of(FileUtil.toSystemDependentName(independentPath)).normalize() + + val prefix = possibleMavenLocalRepositoryRoots.firstOrNull { path.startsWith(it) } + val relativePath = if (prefix != null) { + path.relativeTo(prefix) + } + else { + // allow existing annotation roots on disk to be excluded from downloading + // continue + if (rootType == AnnotationOrderRootType.getInstance() && path.exists()) { + return@mapNotNull null + } + + return CollectResult.Failure( + "Library '${library.name}': root path '$path' does not belong to local maven repository cache " + + "(${possibleMavenLocalRepositoryRoots.distinct().joinToString(", ") { "'$it'" }}).)") + } + + val sha256 = if (libraryProperties.isEnableSha256Checksum) { + val fileUrl = VfsUtil.pathToUrl(path.toString()) + val sha256 = verification[fileUrl]?.sha256sum + + if (sha256 == null && rootType == OrderRootType.CLASSES) { + error("Library '${library.name}': SHA-256 checksum is not specified for url '$fileUrl' in library '${library.name}'.\n" + + "Available checksums:\n" + verification.entries.joinToString("\n") { " ${it.key} -> ${it.value.sha256sum}" }) + } + + sha256 + } + else null + + RelativePathToDownload(relativePath = relativePath, expectedSha256 = sha256) + } + .distinct() + + if (files.isEmpty()) { + return CollectResult.Failure("Library '${library.name}': no roots (files) to download") + } + + return CollectResult.Success(files) + } + + @TestOnly + fun whyLibraryCouldNotBeDownloaded(library: LibraryEx): String? { + val result = collectRelativePathsForJarHttpDownloaderOrLog(library) + return if (result is CollectResult.Failure) result.reason else null + } + } + + private val defaultRetryProvider = RetryProvider.withExponentialBackOff( + System.getProperty("jar.http.downloader.retry.initial.delay.ms", "1000").toLong(), + System.getProperty("jar.http.downloader.retry.backoff.limit.ms", "5000").toLong(), + System.getProperty("jar.http.downloader.retry.max.attempts", "3").toInt(), + ) + + private val NUMBER_OF_DOWNLOAD_THREADS = System.getProperty("jar.http.downloader.threads", "10").toInt() + + @OptIn(ExperimentalCoroutinesApi::class) + private val limitedDispatcher = Dispatchers.IO.limitedParallelism(NUMBER_OF_DOWNLOAD_THREADS) + + private val filesToRefresh = ConcurrentLinkedDeque() + + init { + coroutineScope.launch { + while (true) { + // Refresh every 15 seconds in case fsmonitor events were not handled good enough + delay(15000) + + while (filesToRefresh.isNotEmpty()) { + val file = filesToRefresh.removeFirst() + + if (LOG.isTraceEnabled) { + LOG.trace("Refreshing VFS for file '$file'") + } + + // try to do it as async as possible + // we do not need anything right now, much later is also alright + VfsImplUtil.refreshAndFindFileByPath( + LocalFileSystem.getInstance(), + FileUtil.toSystemIndependentName(file.pathString)) { virtualFile -> + if (virtualFile == null) { + LOG.warn("File '$file' could not be found in VFS after VfsImplUtil.refreshAndFindFileByPath, " + + "exists=${file.exists()}") + } + } + } + } + } + } + + /** + * return null if `library` could not be downloaded by JarHttpDownloader + */ + fun downloadLibraryFilesAsync(library: LibraryEx): Promise<*>? { + val relativePaths = when (val result = collectRelativePathsForJarHttpDownloaderOrLog(library)) { + is CollectResult.Failure -> { + LOG.debug(result.reason) + return null + } + is CollectResult.Success -> result.files + } + + LOG.debug("Downloading library '${library.name}'") + + val localRepository = JarRepositoryManager.getLocalRepositoryPath().toPath() + val remoteRepositories = RemoteRepositoriesConfiguration.getInstance(project).repositories + + // TODO Needs some tests on cancellation, it's not supported well + // it should work both way: cancelling promise should cancel downloading + // and cancelling coroutineScope should cancel promise (make it fail with CancellationException) + + val promise = AsyncPromise() + + coroutineScope.launch { + val remotes = remoteRepositories + .map { repository -> + val authData = obtainAuthenticationData(repository) + RemoteRepository(repository.url, authData) + } + + try { + val downloadedFiles = JarHttpDownloader.downloadLibraryFilesAsync( + relativePaths = relativePaths, + localRepository = localRepository, + remoteRepositories = remotes, + retry = defaultRetryProvider, + downloadDispatcher = limitedDispatcher, + ) + + // Force downloaded files to appear in VFS regardless of fsmonitor + filesToRefresh.addAll(downloadedFiles) + + promise.setResult(Unit) + } + catch (e: Throwable) { + promise.setError(e) + } + } + + return promise + } + + private fun obtainAuthenticationData(repository: RemoteRepositoryDescription): AuthenticationData? { + for (extension in JarRepositoryAuthenticationDataProvider.KEY.extensionList) { + val authData = extension.provideAuthenticationData(repository) + if (authData != null) { + return AuthenticationData(authData.userName, authData.password) + } + } + + return null + } + + sealed interface CollectResult { + data class Success(val files: List) : CollectResult + data class Failure(val reason: String) : CollectResult + } +} diff --git a/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryAuthenticationDataProvider.kt b/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryAuthenticationDataProvider.kt index f43e4dd740de..49655987a6b9 100644 --- a/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryAuthenticationDataProvider.kt +++ b/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryAuthenticationDataProvider.kt @@ -1,9 +1,11 @@ // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.jarRepository +import com.intellij.jarRepository.JarRepositoryAuthenticationDataProvider.AuthenticationData import com.intellij.openapi.Disposable import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.components.Service +import com.intellij.openapi.components.service import com.intellij.openapi.extensions.ExtensionPointName import com.intellij.openapi.util.io.FileUtilRt import com.intellij.openapi.vfs.AsyncFileListener @@ -23,13 +25,19 @@ import org.jetbrains.jps.model.serialization.JpsMavenSettings @ApiStatus.Experimental interface JarRepositoryAuthenticationDataProvider { /** - * @param url url of Maven repository - * @return credentials that allow downloading libraries from the Maven repository located in [url]. + * @param remote Maven repository description + * @return credentials that allow downloading libraries from the specified Maven repository. * [null] if authorization is not needed. */ - fun provideAuthenticationData(url: String): AuthenticationData? + fun provideAuthenticationData(remote: RemoteRepositoryDescription): AuthenticationData? - data class AuthenticationData(val userName: String, val password: String) + data class AuthenticationData(val userName: String, val password: String) { + override fun toString(): String { + return "AuthenticationData(" + + "userName='$userName', " + + "password='${if (password.isEmpty()) "" else "[REDACTED]"}')" + } + } companion object { @JvmField @@ -42,26 +50,30 @@ interface JarRepositoryAuthenticationDataProvider { @RequiresBackgroundThread internal fun obtainAuthenticationData(description: RemoteRepositoryDescription): ArtifactRepositoryManager.ArtifactAuthenticationData? { for (extension in JarRepositoryAuthenticationDataProvider.KEY.extensionList) { - val authData = extension.provideAuthenticationData(description.url) + val authData = extension.provideAuthenticationData(description) if (authData != null) { return ArtifactRepositoryManager.ArtifactAuthenticationData(authData.userName, authData.password) } } - return ApplicationManager.getApplication() - .getService(MavenSettingsXmlRepositoryAuthenticationDataProvider::class.java) - .provideAuthenticationData(description) + return null +} + +class MavenSettingsXmlRepositoryAuthenticationDataProvider: JarRepositoryAuthenticationDataProvider { + override fun provideAuthenticationData(description: RemoteRepositoryDescription): AuthenticationData? { + return service().provideAuthenticationData(description) + } } @Service(Service.Level.APP) -private class MavenSettingsXmlRepositoryAuthenticationDataProvider(private val cs: CoroutineScope) : Disposable { +private class MavenSettingsXmlRepositoryAuthenticationDataService(private val cs: CoroutineScope) : Disposable { private val watchedRoots: List private val globalMavenSettingsXml = JpsMavenSettings.getGlobalMavenSettingsXml() private val userMavenSettingsXml = JpsMavenSettings.getUserMavenSettingsXml() @Volatile - private var cachedAuthentication: Map = emptyMap() + private var cachedAuthentication: Map = emptyMap() init { val localFileSystem = LocalFileSystem.getInstance() @@ -91,7 +103,7 @@ private class MavenSettingsXmlRepositoryAuthenticationDataProvider(private val c reload() } - fun provideAuthenticationData(description: RemoteRepositoryDescription): ArtifactRepositoryManager.ArtifactAuthenticationData? { + fun provideAuthenticationData(description: RemoteRepositoryDescription): AuthenticationData? { return cachedAuthentication[description.id] } @@ -99,7 +111,7 @@ private class MavenSettingsXmlRepositoryAuthenticationDataProvider(private val c cachedAuthentication = JpsMavenSettings.loadAuthenticationSettings(globalMavenSettingsXml, userMavenSettingsXml) .asSequence() .map { (id, authentication) -> - id to ArtifactRepositoryManager.ArtifactAuthenticationData(authentication.username, authentication.password) + id to AuthenticationData(authentication.username, authentication.password) }.toMap() } diff --git a/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryManager.java b/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryManager.java index 376469a087f5..1521f59e32b7 100644 --- a/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryManager.java +++ b/java/idea-ui/src/com/intellij/jarRepository/JarRepositoryManager.java @@ -76,7 +76,8 @@ import static com.intellij.jarRepository.JarRepositoryAuthenticationDataProvider public final class JarRepositoryManager { private static final Logger LOG = Logger.getInstance(JarRepositoryManager.class); - private static final String MAVEN_REPOSITORY_MACRO = "$MAVEN_REPOSITORY$"; + static final String MAVEN_REPOSITORY_MACRO = "$MAVEN_REPOSITORY$"; + private static final String DEFAULT_REPOSITORY_PATH = ".m2/repository"; private static final AtomicInteger ourTasksInProgress = new AtomicInteger(); diff --git a/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibrarySynchronizer.kt b/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibrarySynchronizer.kt index 766573d7029a..73f51540b731 100644 --- a/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibrarySynchronizer.kt +++ b/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibrarySynchronizer.kt @@ -85,15 +85,16 @@ internal fun collectLibraries(project: Project, predicate: (Library) -> Boolean) return result } -internal fun isLibraryNeedToBeReloaded(library: LibraryEx, properties: RepositoryLibraryProperties): Boolean { +internal fun isLibraryHasFixedVersion(properties: RepositoryLibraryProperties): Boolean { val version = properties.version ?: return false - if (version == RepositoryLibraryDescription.LatestVersionId || - version == RepositoryLibraryDescription.ReleaseVersionId || - version.endsWith(RepositoryLibraryDescription.SnapshotVersionSuffix)) { - return true - } + return version != RepositoryLibraryDescription.LatestVersionId && + version != RepositoryLibraryDescription.ReleaseVersionId && + !version.endsWith(RepositoryLibraryDescription.SnapshotVersionSuffix) +} - return OrderRootType.getAllTypes().any { library.getFiles(it).size != library.getUrls(it).size } +internal fun isLibraryNeedToBeReloaded(library: LibraryEx, properties: RepositoryLibraryProperties): Boolean { + return !isLibraryHasFixedVersion(properties) || + OrderRootType.getAllTypes().any { library.getFiles(it).size != library.getUrls(it).size } } internal fun removeDuplicatedUrlsFromRepositoryLibraries(project: Project) { diff --git a/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibraryUtils.kt b/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibraryUtils.kt index 9c1e81219010..a1573847f930 100644 --- a/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibraryUtils.kt +++ b/java/idea-ui/src/com/intellij/jarRepository/RepositoryLibraryUtils.kt @@ -13,10 +13,11 @@ import com.intellij.openapi.application.EDT import com.intellij.openapi.application.WriteAction import com.intellij.openapi.components.Service import com.intellij.openapi.components.service +import com.intellij.openapi.components.serviceAsync import com.intellij.openapi.diagnostic.logger import com.intellij.openapi.project.Project -import com.intellij.openapi.roots.JavadocOrderRootType import com.intellij.openapi.roots.OrderRootType +import com.intellij.openapi.roots.impl.libraries.LibraryEx import com.intellij.openapi.roots.libraries.ui.OrderRoot import com.intellij.openapi.roots.ui.configuration.libraryEditor.LibraryEditor import com.intellij.openapi.util.JDOMUtil @@ -29,20 +30,22 @@ import com.intellij.platform.util.progress.RawProgressReporter import com.intellij.platform.util.progress.reportRawProgress import com.intellij.platform.workspace.jps.entities.* import com.intellij.platform.workspace.storage.EntityStorage +import com.intellij.platform.workspace.storage.ImmutableEntityStorage import com.intellij.platform.workspace.storage.MutableEntityStorage import com.intellij.platform.workspace.storage.instrumentation.EntityStorageInstrumentationApi import com.intellij.platform.workspace.storage.instrumentation.MutableEntityStorageInstrumentation import com.intellij.platform.workspace.storage.toBuilder import com.intellij.util.containers.ContainerUtil +import com.intellij.workspaceModel.ide.impl.legacyBridge.library.findLibraryBridge import kotlinx.coroutines.* import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.TestOnly import org.jetbrains.concurrency.await import org.jetbrains.concurrency.collectResults -import org.jetbrains.concurrency.rejectedPromise import org.jetbrains.concurrency.resolvedPromise import org.jetbrains.idea.maven.utils.library.RepositoryLibraryDescription import org.jetbrains.idea.maven.utils.library.RepositoryLibraryProperties +import org.jetbrains.idea.maven.utils.library.RepositoryUtils import org.jetbrains.jps.model.library.JpsMavenRepositoryLibraryDescriptor import org.jetbrains.jps.model.library.JpsMavenRepositoryLibraryDescriptor.ArtifactVerification import org.jetbrains.jps.model.serialization.library.JpsLibraryTableSerializer @@ -139,9 +142,9 @@ class RepositoryLibraryUtils(private val project: Project, private val cs: Corou fun resolveAllBackground(): Deferred { return runBackground(JavaUiBundle.message("repository.library.utils.progress.title.resolving.all.libraries")) { reportRawProgress { reporter -> - val snapshot = WorkspaceModel.getInstance(project).currentSnapshot + val snapshot = project.serviceAsync().currentSnapshot val libraries = snapshot.entities(LibraryEntity::class.java).toList() - val failedToResolve = resolve(reporter, libraries.asSequence()) + val failedToResolve = resolve(snapshot, reporter, libraries.asSequence()) if (failedToResolve.isEmpty()) { showNotification(JavaUiBundle.message("repository.library.utils.notification.content.libraries.resolve.success", libraries.size), NotificationType.INFORMATION) @@ -202,7 +205,7 @@ class RepositoryLibraryUtils(private val project: Project, private val cs: Corou /** * @return A list of library entities failed to resolve or an empty list if resolution is successful. */ - private suspend fun resolve(reporter: RawProgressReporter, libraries: Sequence): List = coroutineScope { + private suspend fun resolve(snapshot: ImmutableEntityStorage, reporter: RawProgressReporter, libraries: Sequence): List = coroutineScope { val librariesAsList = libraries.toList() if (librariesAsList.isEmpty()) return@coroutineScope emptyList() @@ -212,19 +215,14 @@ class RepositoryLibraryUtils(private val project: Project, private val cs: Corou val failedList: MutableList = CopyOnWriteArrayList() librariesAsList.map { lib -> - val entity = lib.libraryProperties ?: return@map resolvedPromise() - val properties = entity.toRepositoryLibraryProperties() ?: return@map resolvedPromise() + lib.libraryProperties ?: return@map resolvedPromise() - val downloadSources = lib.roots.any { it.type == LibraryRootTypeId.SOURCES } - val downloadJavadoc = lib.roots.any { it.type.name == JavadocOrderRootType.getInstance().name() } - - val promise = - JarRepositoryManager.loadDependenciesAsync(project, properties, downloadSources, downloadJavadoc, null, null).thenAsync { - if (it == null || it.isEmpty()) rejectedPromise() else resolvedPromise(it) - } + // the correct way is to move RepositoryUtils from LibraryEx to LibraryEntity + val libEx = lib.findLibraryBridge(snapshot) as LibraryEx + val promise = RepositoryUtils.reloadDependencies(project, libEx) promise.onError { failedList.add(lib) } promise.onProcessed { reporter.updateProgressDetailsFraction(completeCounter, total) } - promise + promise.then { } }.collectResults(ignoreErrors = true).await() // We're collecting errors manually, fail silently reporter.resetProgressDetailsFraction() @@ -360,7 +358,7 @@ class RepositoryLibraryUtils(private val project: Project, private val cs: Corou } .map { (entity, _) -> entity } - val unresolvedBeforeUpdate = resolve(reporter, preResolveEntities.map { it.library }) + val unresolvedBeforeUpdate = resolve(snapshot, reporter, preResolveEntities.map { it.library }) if (unresolvedBeforeUpdate.isNotEmpty()) { logger.info("Building libraries properties progressed: resolving before update failed, cancelling operation") showFailedToResolveNotification(unresolvedBeforeUpdate, snapshot) { shown, notShownSize -> @@ -427,7 +425,7 @@ class RepositoryLibraryUtils(private val project: Project, private val cs: Corou snapshotAfterUpdate.resolve(entity.library.symbolicId)?.libraryProperties }.asSequence() - val unresolvedAfterUpdate = resolve(reporter, entitiesAfterUpdate.map { it.library }) + val unresolvedAfterUpdate = resolve(snapshot, reporter, entitiesAfterUpdate.map { it.library }) showUpdateCompleteNotification(disableInfoNotifications, updatedEntitiesAndProperties.size, snapshotAfterUpdate, diff --git a/java/idea-ui/src/com/intellij/jarRepository/settings/repositoryLibrariesReloader.kt b/java/idea-ui/src/com/intellij/jarRepository/settings/repositoryLibrariesReloader.kt index a0ea16d8d1b3..253bcbd7424a 100644 --- a/java/idea-ui/src/com/intellij/jarRepository/settings/repositoryLibrariesReloader.kt +++ b/java/idea-ui/src/com/intellij/jarRepository/settings/repositoryLibrariesReloader.kt @@ -17,7 +17,9 @@ internal fun reloadAllRepositoryLibraries(project: Project) { libraries .asSequence() .filterIsInstance() - .map { RepositoryUtils.reloadDependencies(project, it) } + .map { RepositoryUtils.reloadDependencies(project, it).then { + // NOP, to make collectResults accept the correct type (Unit) + } } .toList() .collectResults() .onSuccess { diff --git a/java/idea-ui/src/org/jetbrains/idea/maven/utils/library/RepositoryUtils.java b/java/idea-ui/src/org/jetbrains/idea/maven/utils/library/RepositoryUtils.java index f610c52f80c9..db84968ae50a 100644 --- a/java/idea-ui/src/org/jetbrains/idea/maven/utils/library/RepositoryUtils.java +++ b/java/idea-ui/src/org/jetbrains/idea/maven/utils/library/RepositoryUtils.java @@ -1,6 +1,7 @@ // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package org.jetbrains.idea.maven.utils.library; +import com.intellij.jarRepository.JarHttpDownloaderJps; import com.intellij.jarRepository.JarRepositoryManager; import com.intellij.jarRepository.RepositoryLibraryType; import com.intellij.openapi.application.ApplicationManager; @@ -210,11 +211,38 @@ public final class RepositoryUtils { return true; } - public static Promise> reloadDependencies(@NotNull final Project project, @NotNull final LibraryEx library) { - return loadDependenciesToLibrary(project, library, libraryHasSources(library), libraryHasJavaDocs(library), getStorageRoot(library)); + public static Promise reloadDependencies(@NotNull final Project project, @NotNull final LibraryEx library) { + if (JarHttpDownloaderJps.enabled()) { + Promise promise = JarHttpDownloaderJps.getInstance(project).downloadLibraryFilesAsync(library); + + // null means this library should be handled by standard resolver + if (promise != null) { + // callers of this function typically do not log, so do it for them + promise.onError(error -> { + LOG.warn("Failed to download repository library '" + library.getName() + "' with JarHttpDownloader", error); + }); + + if (LOG.isDebugEnabled()) { + promise.onSuccess(result -> { + LOG.debug("Downloaded repository library '" + library.getName() + "' with JarHttpDownloader"); + }); + } + + return promise; + } + } + + Promise> mavenResolverPromise = loadDependenciesToLibrary( + project, library, libraryHasSources(library), libraryHasJavaDocs(library), getStorageRoot(library)); + // callers of this function typically do not log, so do it for them + mavenResolverPromise.onError(error -> { + LOG.warn("Failed to download repository library '" + library.getName() + "' with maven resolver", error); + }); + + return mavenResolverPromise; } - public static Promise> deleteAndReloadDependencies(@NotNull final Project project, + public static Promise deleteAndReloadDependencies(@NotNull final Project project, @NotNull final LibraryEx library) throws IOException { LOG.debug("start deleting files in library " + library.getName()); var filesToDelete = new ArrayList(); diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/.gitignore b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/.gitignore new file mode 100644 index 000000000000..b0568b7ae9c0 --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/.gitignore @@ -0,0 +1,9 @@ +# Default ignored files +/shelf/ +/workspace.xml +/vcs.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/jarRepositories.xml b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/jarRepositories.xml new file mode 100644 index 000000000000..41ee8f079524 --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/jarRepositories.xml @@ -0,0 +1,10 @@ + + + + + + + \ No newline at end of file diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/libraries/apache_commons_math3.xml b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/libraries/apache_commons_math3.xml new file mode 100644 index 000000000000..2d45ed95c1b5 --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/libraries/apache_commons_math3.xml @@ -0,0 +1,18 @@ + + + + + + 79b0baf88d2bc643f652f413e52702d81ac40a9b782d7f00fc431739e8d1c28a + + + + + + + + + + + + \ No newline at end of file diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/libraries/apache_commons_math3_bad_checksum.xml b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/libraries/apache_commons_math3_bad_checksum.xml new file mode 100644 index 000000000000..9a9c9cb5806a --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/libraries/apache_commons_math3_bad_checksum.xml @@ -0,0 +1,18 @@ + + + + + + 000000000000000000000000000000000000000000000000000000000000028a + + + + + + + + + + + + \ No newline at end of file diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/misc.xml b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/misc.xml new file mode 100644 index 000000000000..a6e1098c7d41 --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/modules.xml b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/modules.xml new file mode 100644 index 000000000000..f5a0f608dc0e --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/src/Main.java b/java/idea-ui/testData/testProjectJarHttpDownloader/src/Main.java new file mode 100644 index 000000000000..962de4ede956 --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/src/Main.java @@ -0,0 +1,5 @@ +public class Main { + public static void main(String[] args) { + System.out.println("Hello world!"); + } +} \ No newline at end of file diff --git a/java/idea-ui/testData/testProjectJarHttpDownloader/testProject.iml b/java/idea-ui/testData/testProjectJarHttpDownloader/testProject.iml new file mode 100644 index 000000000000..c90834f2d607 --- /dev/null +++ b/java/idea-ui/testData/testProjectJarHttpDownloader/testProject.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderArtifactTest.kt b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderArtifactTest.kt new file mode 100644 index 000000000000..8c2698482b1d --- /dev/null +++ b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderArtifactTest.kt @@ -0,0 +1,246 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.jarRepository + +import com.intellij.jarRepository.JarHttpDownloader.RemoteRepository +import com.intellij.jarRepository.JarHttpDownloader.downloadArtifact +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.createContext +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.url +import com.intellij.jarRepository.JarRepositoryAuthenticationDataProvider.AuthenticationData +import com.intellij.testFramework.rules.TempDirectoryExtension +import com.intellij.util.io.DigestUtil +import com.intellij.util.io.HttpRequests +import io.ktor.http.HttpStatusCode +import io.ktor.server.engine.ApplicationEngine +import org.jetbrains.idea.maven.aether.RetryProvider +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import java.io.IOException +import java.net.InetAddress +import java.net.ServerSocket +import java.nio.file.Path +import kotlin.io.path.readText +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class JarHttpDownloaderArtifactTest { + @JvmField + @RegisterExtension + internal val serverExtension = JarHttpDownloaderTestUtil.TestHttpServerExtension() + private val server: ApplicationEngine get() = serverExtension.server + + @JvmField + @RegisterExtension + internal val tempDirectory = TempDirectoryExtension() + + private val localRepository by lazy { + tempDirectory.newDirectoryPath("local") + } + + private val remoteRepositories by lazy { + listOf( + RemoteRepository(server.url + "/a", null), + RemoteRepository(server.url + "/b", AuthenticationData("u", "pass")), + RemoteRepository(server.url + "/c", null), + ) + } + + @BeforeEach + fun setUp() { + JarHttpDownloader.forceHttps = false + + } + + @AfterEach + fun tearDown() { + JarHttpDownloader.forceHttps = true + } + + @Test + fun downloadArtifact_second_succeed() { + server.createContext("/b/c/file.data", HttpStatusCode.OK, response = "Hello, world!") + + val localFile = downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), null), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + ) + + assertEquals(localRepository.resolve("c/file.data"), localFile) + assertEquals("Hello, world!", localFile.readText()) + + assertEquals(""" + /a/c/file.data: 404 + /b/c/file.data: 200 + """.trimIndent(), serverExtension.log.trim()) + } + + @Test + fun downloadArtifact_authenticate() { + server.createContext("/b/c/file.data", HttpStatusCode.OK, response = "Hello, world!", auth = AuthenticationData("u", "pass")) + + val localFile = downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), null), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + ) + + assertEquals(localRepository.resolve("c/file.data"), localFile) + assertEquals("Hello, world!", localFile.readText()) + } + + @Test + fun downloadArtifact_authenticate_wrong_password() { + server.createContext("/b/c/file.data", HttpStatusCode.OK, response = "Hello, world!", auth = AuthenticationData("u", "another password")) + + val e = assertFailsWith { + downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), null), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + ) + } + + assertEquals("Artifact 'c/file.data' was not found in remote repositories, some of them returned 401 Unauthorized: [${server.url}/b/c/file.data]", e.message) + val suppressed = e.suppressedExceptions.single() as HttpRequests.HttpStatusException + assertEquals("Request failed with status code 401", suppressed.message) + assertEquals("${server.url}/b/c/file.data", suppressed.url) + } + + @Test + fun downloadArtifact_skip_wrong_authentication() { + // repository 'a': artifact missing + // repository 'b': wrong authentication + // repository 'c': OK + server.createContext("/b/c/file.data", HttpStatusCode.OK, response = "secret data", auth = AuthenticationData("u", "another password")) + server.createContext("/c/c/file.data", HttpStatusCode.OK, response = "Hello, world!") + + downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), null), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + ) + + assertEquals("Hello, world!", localRepository.resolve("c/file.data").readText()) + + assertEquals(""" + /a/c/file.data: 404 + /b/c/file.data: 401 + /c/c/file.data: 200 + """.trimIndent(), serverExtension.log.trim()) + } + + @Test + fun downloadArtifact_do_not_skip_internal_server_error() { + // repository 'a': artifact missing + // repository 'b': 500 + // ... we should not query 'c' for security reasons (when checksum missing, we can't guarantee what we'll find in 'c') + // + // case when 'expectedSha256!=null' was not implemented (we may query everything then), but could be + + server.createContext("/b/c/file.data", HttpStatusCode.InternalServerError) + + val e = assertFailsWith { + downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), null), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + ) + } + + assertEquals("Request failed with status code 500", e.message) + assertEquals("${server.url}/b/c/file.data", e.url) + + assertEquals(""" + /a/c/file.data: 404 + /b/c/file.data: 500 + """.trimIndent(), serverExtension.log.trim()) + } + + @Test + fun downloadArtifact_do_not_skip_connection_refused() { + // repository 'a': artifact missing + // repository 'b': connection refused + // ... we should not query 'c' for security reasons (when checksum missing, we can't guarantee what we'll find in 'c') + // + // case when 'expectedSha256!=null' was not implemented (we may query everything then), but could be + + val bindAddr = InetAddress.getByName("127.0.0.1") + val unusedLocalPort = ServerSocket(0, 10, bindAddr).use { it.localPort } + + val e = assertFailsWith { + downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), null), + localRepository = localRepository, + remoteRepositories = listOf( + RemoteRepository(server.url + "/a", null), + RemoteRepository("http://127.0.0.1:${unusedLocalPort}/root", null), + RemoteRepository(server.url + "/c", null), + ), + retry = RetryProvider.disabled(), + ) + } + + assertTrue(e.message!!.contains("http://127.0.0.1:${unusedLocalPort}/root/c/file.data"), e.message) + assertTrue(e.message!!.contains("Connection refused"), e.message) + + assertEquals(""" + /a/c/file.data: 404 + """.trimIndent(), serverExtension.log.trim()) + } + + @Test + fun downloadArtifact_wrong_checksum() { + val response = "Hello, world!" + server.createContext("/b/c/file.data", HttpStatusCode.OK, response = response) + + val actualSha256 = DigestUtil.sha256Hex(response.toByteArray()) + val expectedSha256 = DigestUtil.sha256Hex("wrong".toByteArray()) + + val e = assertFailsWith { + downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), expectedSha256), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + ) + } + + assertTrue(e.message!!.startsWith("Wrong file checksum after downloading '${server.url}/b/c/file.data'"), e.message) + assertTrue(e.message!!.contains(expectedSha256), e.message) + assertTrue(e.message!!.contains(actualSha256), e.message) + assertTrue(e.message!!.contains("(fileSize: 13)"), e.message) + + assertEquals(""" + /a/c/file.data: 404 + /b/c/file.data: 200 + """.trimIndent(), serverExtension.log.trim()) + } + + @Test + fun downloadArtifact_missing_artifact() { + val e = assertFailsWith { + downloadArtifact( + artifactPath = JarHttpDownloader.RelativePathToDownload(Path.of("c/file.data"), null), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + ) + } + + assertEquals("Artifact 'c/file.data' was not found in remote repositories: [${server.url}/a/c/file.data, ${server.url}/b/c/file.data, ${server.url}/c/c/file.data]", e.message) + + assertEquals(""" + /a/c/file.data: 404 + /b/c/file.data: 404 + /c/c/file.data: 404 + """.trimIndent(), serverExtension.log.trim()) + } +} \ No newline at end of file diff --git a/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderFileTest.kt b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderFileTest.kt new file mode 100644 index 000000000000..bd4a62ef49d0 --- /dev/null +++ b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderFileTest.kt @@ -0,0 +1,213 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.jarRepository + +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.TestHttpServerExtension +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.createContext +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.url +import com.intellij.openapi.diagnostic.Logger +import com.intellij.testFramework.rules.TempDirectoryExtension +import com.intellij.util.io.HttpRequests +import com.intellij.util.io.sha256Hex +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.OutgoingContent +import io.ktor.server.application.call +import io.ktor.server.engine.ApplicationEngine +import io.ktor.server.response.respond +import io.ktor.server.routing.get +import io.ktor.server.routing.routing +import org.jetbrains.idea.maven.aether.Retry +import org.jetbrains.idea.maven.aether.RetryProvider +import org.jetbrains.idea.maven.aether.ThrowingSupplier +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import java.io.IOException +import java.net.InetAddress +import java.net.ServerSocket +import java.util.concurrent.atomic.AtomicInteger +import kotlin.io.path.exists +import kotlin.io.path.fileSize +import kotlin.io.path.readText +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class JarHttpDownloaderFileTest { + @JvmField + @RegisterExtension + internal val serverExtension = TestHttpServerExtension { server -> + server.createContext("/demo.data", HttpStatusCode.OK, response = "Hello, world!") + } + private val server: ApplicationEngine get() = serverExtension.server + + @JvmField + @RegisterExtension + val tempDirectory = TempDirectoryExtension() + + @BeforeEach + fun setUp() { + JarHttpDownloader.forceHttps = false + } + + @AfterEach + fun tearDown() { + JarHttpDownloader.forceHttps = true + } + + @Test + fun downloadFile_force_https() { + JarHttpDownloader.forceHttps = true + val file = tempDirectory.rootPath.resolve("ant.pom") + val url = server.url + "/demo.data" + val exception = assertFailsWith { + JarHttpDownloader.downloadFile(url, file, RetryProvider.disabled()) + } + assertEquals("Url must have https protocol: $url", exception.message) + } + + @Test + fun downloadFile_retries() { + val log = StringBuilder() + + val retry = RetryProvider.withExponentialBackOff(1, 1, 3) + + for (code in listOf(HttpStatusCode.Unauthorized, HttpStatusCode.NotFound, HttpStatusCode.InternalServerError)) { + val url = server.url + "/code-${code.value}" + server.createContext("/${url.substringAfterLast('/')}", code, response = "Hello, world!", log = { + log.appendLine("reply with $code") + }) + + val file = tempDirectory.rootPath.resolve("x") + val exception = assertFailsWith { + JarHttpDownloader.downloadFile(url, file, retry = retry) + } + assertEquals(code.value, exception.statusCode) + } + + assertEquals(""" + /code-401: 401 + /code-404: 404 + /code-500: 500 + /code-500: 500 + /code-500: 500 + """.trimIndent(), serverExtension.log.trim()) + } + + @Test + fun downloadFile_retry_recovery() { + val log = StringBuilder() + + val retry = RetryProvider.withExponentialBackOff(1, 1, 3) + + val url = server.url + "/x" + val countOfInternalServerErrors = AtomicInteger(2) + server.application.routing { + get("/${url.substringAfterLast('/')}") { + val code = if (countOfInternalServerErrors.decrementAndGet() >= 0) HttpStatusCode.InternalServerError else HttpStatusCode.OK + call.respond(code, "Hello, world!") + log.appendLine("reply with ${code.value}") + } + } + + val file = tempDirectory.rootPath.resolve("x") + JarHttpDownloader.downloadFile(url, file, retry = retry) + assertEquals("Hello, world!", file.readText()) + + assertEquals(""" + reply with 500 + reply with 500 + reply with 200 + """.trimIndent(), log.toString().trim()) + } + + @Test + fun downloadFile_retry_connection_refused() { + val bindAddr = InetAddress.getByName("127.0.0.1") + val serverPort = ServerSocket(0, 10, bindAddr).use { it.localPort } + + val attempts = AtomicInteger(0) + + val originalRetry = RetryProvider.withExponentialBackOff(1, 1, 3) + val retry = object : Retry { + override fun retry(supplier: ThrowingSupplier, logger: Logger): R? { + return originalRetry.retry(ThrowingSupplier { + attempts.incrementAndGet() + supplier.get() + }, logger) + } + } + + val url = "http://127.0.0.1:${serverPort}/x" + val exception = assertFailsWith { + JarHttpDownloader.downloadFile(url, tempDirectory.rootPath.resolve("x"), retry = retry) + } + + assertTrue(exception.message!!.contains("Connection refused"), exception.message) + assertEquals(3, attempts.get()) + } + + @Test + fun downloadFile_fail_on_missing_content_length() { + server.application.routing { + get("/unknown-content-length.data") { + call.respond(object : OutgoingContent.ByteArrayContent() { + override fun bytes(): ByteArray = "Hello, world!".toByteArray() + override val contentLength: Long? = null + }) + } + } + + val file = tempDirectory.rootPath.resolve("x") + val url = server.url + "/unknown-content-length.data" + val exception = assertFailsWith { + JarHttpDownloader.downloadFile(url, file, RetryProvider.disabled()) + } + assertEquals("Header 'Content-Length' is missing or zero for $url", exception.message) + } + + @Test + fun downloadFile_no_expected_checksum() { + val file = tempDirectory.rootPath.resolve("ant.pom") + JarHttpDownloader.downloadFile(server.url + "/demo.data", file, RetryProvider.disabled()) + assertEquals(13, file.fileSize()) + assertEquals("315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3", sha256Hex(file)) + } + + @Test + fun downloadFile_expected_checksum() { + val file = tempDirectory.rootPath.resolve("ant.pom") + JarHttpDownloader.downloadFile( + server.url + "/demo.data", file, RetryProvider.disabled(), + expectedSha256 = "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3") + assertEquals(13, file.fileSize()) + assertEquals("315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3", sha256Hex(file)) + } + + @Test + fun downloadFile_wrong_checksum() { + val attempts = AtomicInteger(0) + + val originalRetry = RetryProvider.withExponentialBackOff(1, 1, 3) + val retry = object : Retry { + override fun retry(supplier: ThrowingSupplier, logger: Logger): R? { + return originalRetry.retry(ThrowingSupplier { + attempts.incrementAndGet() + supplier.get() + }, logger) + } + } + + val file = tempDirectory.rootPath.resolve("ant.pom") + val url = server.url + "/demo.data" + val exception = assertFailsWith { + JarHttpDownloader.downloadFile( + url, file, retry, + expectedSha256 = "b6276017cf6f2a07b7b7b62778333237ca73405fcc3af1ca9d95f52f97fb7000") + } + assertFalse(file.exists()) + assertTrue(exception.message!!.startsWith("Wrong file checksum after downloading"), exception.message) + assertEquals(1, attempts.get(), "Only one attempt should be made") + } +} \ No newline at end of file diff --git a/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderJpsTest.kt b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderJpsTest.kt new file mode 100644 index 000000000000..5178fd1e24a6 --- /dev/null +++ b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderJpsTest.kt @@ -0,0 +1,181 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.jarRepository + +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.TestHttpServerExtension +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.createContext +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.url +import com.intellij.openapi.application.PathMacros +import com.intellij.openapi.application.ex.PathManagerEx.findFileUnderCommunityHome +import com.intellij.openapi.project.Project +import com.intellij.openapi.project.doNotEnableExternalStorageByDefaultInTests +import com.intellij.openapi.roots.impl.libraries.LibraryEx +import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar +import com.intellij.openapi.util.Disposer +import com.intellij.openapi.util.io.FileUtil +import com.intellij.openapi.vfs.VfsUtil +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.testFramework.TemporaryDirectoryExtension +import com.intellij.testFramework.createOrLoadProject +import com.intellij.testFramework.junit5.TestApplication +import io.ktor.http.HttpStatusCode +import io.ktor.server.engine.ApplicationEngine +import kotlinx.coroutines.runBlocking +import org.jetbrains.concurrency.await +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.readText +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@TestApplication +class JarHttpDownloaderJpsTest { + private val TEST_MAVEN_LOCAL_REPOSITORY_MACRO = "REPOSITORY_LIBRARY_UTILS_TEST_LOCAL_MAVEN_REPOSITORY" + private val TEST_REMOTE_REPOSITORIES_ROOT_MACRO = "REPOSITORY_LIBRARY_UTILS_TEST_REMOTE_REPOSITORIES_ROOT" + + @JvmField + @RegisterExtension + val m2DirectoryExtension = TemporaryDirectoryExtension() + + private val m2DirectoryPath by lazy { m2DirectoryExtension.createDir() } + + private val authUsername = "user" + @Suppress("SpellCheckingInspection") + private val authPassword = "passw0rd" + + @RegisterExtension + @JvmField + internal val serverExtension = TestHttpServerExtension { server -> + server.createContext("/", HttpStatusCode.NotFound) + + server.createContext( + path = "/org/apache/commons/commons-math3/3.6/commons-math3-3.6-sources.jar", + httpStatusCode = HttpStatusCode.OK, + response = "fake sources jar content", + auth = JarRepositoryAuthenticationDataProvider.AuthenticationData(authUsername, authPassword), + ) + + server.createContext( + path = "/org/apache/commons/commons-math3/3.6/commons-math3-3.6.jar", + httpStatusCode = HttpStatusCode.OK, + response = "fake jar content", + auth = JarRepositoryAuthenticationDataProvider.AuthenticationData(authUsername, authPassword), + ) + } + private val server: ApplicationEngine get() = serverExtension.server + + private val disposable = Disposer.newDisposable(javaClass.name) + + @BeforeEach + fun beforeEach() { + val pathMacros: PathMacros = PathMacros.getInstance() + pathMacros.setMacro(TEST_MAVEN_LOCAL_REPOSITORY_MACRO, m2DirectoryPath.toString()) + Disposer.register(disposable) { + pathMacros.setMacro(TEST_MAVEN_LOCAL_REPOSITORY_MACRO, null) + } + + pathMacros.setMacro(TEST_REMOTE_REPOSITORIES_ROOT_MACRO, server.url) + Disposer.register(disposable) { + pathMacros.setMacro(TEST_REMOTE_REPOSITORIES_ROOT_MACRO, null) + } + + JarRepositoryManager.setLocalRepositoryPath(m2DirectoryPath.toFile()) + Disposer.register(disposable) { + JarRepositoryManager.setLocalRepositoryPath(null) + } + + assertTrue(JarHttpDownloader.forceHttps, "default forceHttps must be true") + JarHttpDownloader.forceHttps = false + Disposer.register(disposable) { + JarHttpDownloader.forceHttps = true + } + + JarRepositoryAuthenticationDataProvider.KEY.point.registerExtension(object : JarRepositoryAuthenticationDataProvider { + override fun provideAuthenticationData(description: RemoteRepositoryDescription): JarRepositoryAuthenticationDataProvider.AuthenticationData? = when (description.url) { + server.url -> JarRepositoryAuthenticationDataProvider.AuthenticationData(authUsername, authPassword) + else -> null + } + + }, disposable) + } + + @AfterEach + fun afterEach() { + Disposer.dispose(disposable) + } + + @RegisterExtension + @JvmField + val projectDirectory = TemporaryDirectoryExtension() + + @Test + fun happy_case() = testRepositoryLibraryUtils(projectTestData) { project, utils -> + val libraryRelease = getLibrary(project, "apache.commons.math3") as LibraryEx + val promise = JarHttpDownloaderJps.getInstance(project).downloadLibraryFilesAsync(libraryRelease) + promise!!.await() + + val jar = m2DirectoryPath.resolve("org/apache/commons/commons-math3/3.6/commons-math3-3.6.jar") + assertEquals("fake jar content", jar.readText()) + + val sourcesJar = m2DirectoryPath.resolve("org/apache/commons/commons-math3/3.6/commons-math3-3.6-sources.jar") + assertEquals("fake sources jar content", sourcesJar.readText()) + } + + @Test + fun bad_checksum() = testRepositoryLibraryUtils(projectTestData) { project, utils -> + val libraryRelease = getLibrary(project, "apache.commons.math3.bad.checksum") as LibraryEx + val promise = JarHttpDownloaderJps.getInstance(project).downloadLibraryFilesAsync(libraryRelease) + + val exception = assertFailsWith { + promise!!.await() + } + + assertTrue(exception.message!!.startsWith("Failed to download 1 artifact(s): (first exception) Wrong file checksum"), exception.message!!) + + val cause = exception.cause!! + assertTrue(cause.message!!.contains("Wrong file checksum after downloading '${server.url}/org/apache/commons/commons-math3/3.6/commons-math3-3.6.jar'"), cause.message!!) + assertTrue(cause.message!!.contains("expected checksum 000000000000000000000000000000000000000000000000000000000000028a, but got 79b0baf88d2bc643f652f413e52702d81ac40a9b782d7f00fc431739e8d1c28a"), cause.message!!) + + // not downloaded due to wrong checksum + val jar = m2DirectoryPath.resolve("org/apache/commons/commons-math3/3.6/commons-math3-3.6.jar") + Assertions.assertFalse(jar.exists()) + + // still downloaded + val sourcesJar = m2DirectoryPath.resolve("org/apache/commons/commons-math3/3.6/commons-math3-3.6-sources.jar") + assertEquals("fake sources jar content", sourcesJar.readText()) + } + + private fun getLibrary(project: Project, name: String) = + LibraryTablesRegistrar.getInstance() + .getLibraryTable(project) + .getLibraryByName(name) + ?: error("Library '$name' was not found in ${project.basePath}") + + private val projectTestData = findFileUnderCommunityHome("java/idea-ui/testData/testProjectJarHttpDownloader").toPath() + + private fun testRepositoryLibraryUtils(sampleProjectPath: Path, checkProject: suspend (Project, RepositoryLibraryUtils) -> Unit) { + fun copyProjectFiles(dir: VirtualFile): Path { + val projectDir = dir.toNioPath() + FileUtil.copyDir(sampleProjectPath.toFile(), projectDir.toFile()) + VfsUtil.markDirtyAndRefresh(false, true, true, dir) + return projectDir + } + + doNotEnableExternalStorageByDefaultInTests { + runBlocking { + createOrLoadProject(projectDirectory, ::copyProjectFiles, loadComponentState = true, useDefaultProjectSettings = false) { project -> + val utils = RepositoryLibraryUtils.getInstance(project) + utils.setTestCoroutineScope(this) + checkProject(project, utils) + utils.resetTestCoroutineScope() + } + } + } + } + +} \ No newline at end of file diff --git a/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderLibraryFilesTest.kt b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderLibraryFilesTest.kt new file mode 100644 index 000000000000..2f10c7eb9168 --- /dev/null +++ b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderLibraryFilesTest.kt @@ -0,0 +1,116 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.jarRepository + +import com.intellij.jarRepository.JarHttpDownloader.RemoteRepository +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.TestHttpServerExtension +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.createContext +import com.intellij.jarRepository.JarHttpDownloaderTestUtil.url +import com.intellij.testFramework.rules.TempDirectoryExtension +import io.ktor.http.HttpStatusCode +import io.ktor.server.engine.ApplicationEngine +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import org.jetbrains.idea.maven.aether.RetryProvider +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import org.junit.jupiter.api.fail +import java.nio.file.Path +import kotlin.io.path.readText +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class JarHttpDownloaderLibraryFilesTest { + @JvmField + @RegisterExtension + internal val serverExtension = TestHttpServerExtension() + private val server: ApplicationEngine get() = serverExtension.server + + @JvmField + @RegisterExtension + internal val tempDirectory = TempDirectoryExtension() + + private val localRepository by lazy { + tempDirectory.newDirectoryPath("local") + } + + private val remoteRepositories by lazy { + listOf( + RemoteRepository(server.url + "/a", null), + ) + } + + @BeforeEach + fun setUp() { + JarHttpDownloader.forceHttps = false + } + + @AfterEach + fun tearDown() { + JarHttpDownloader.forceHttps = true + } + + // test that we wait for all downloads even if one of the downloads fails + // also tests that we handle connections in parallel + @Test + fun downloadLibraryFilesAsync_finish_all_files() { + val response = "data" + + server.createContext("/a/fail.data", HttpStatusCode.NotFound, response = response) + server.createContext("/a/delay.data", HttpStatusCode.OK, response = response, delayMs = 500) + + runBlocking(Dispatchers.IO) { + val failure = assertFailsWith { + JarHttpDownloader.downloadLibraryFilesAsync( + relativePaths = listOf( + JarHttpDownloader.RelativePathToDownload(Path.of("fail.data"), null), + JarHttpDownloader.RelativePathToDownload(Path.of("delay.data"), null), + ), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + downloadDispatcher = Dispatchers.IO, + ) + } + + val message = "Failed to download 1 artifact(s): (first exception) " + + "Artifact 'fail.data' was not found in remote repositories: [${server.url}/a/fail.data]" + if (failure.message != message) { + fail( + "Expected message: '$message'\n" + + "Actual message: '${failure.message}'\n" + + "Stacktrace:\n" + + failure.stackTraceToString()) + } + + assertEquals("Artifact 'fail.data' was not found in remote repositories: [${server.url}/a/fail.data]", failure.cause!!.message) + + assertEquals(response, localRepository.resolve("delay.data").readText()) + } + + assertEquals(""" + /a/fail.data: 404 + /a/delay.data: 200 + """.trimIndent(), serverExtension.log.trim()) + } + + @Test + fun downloadLibraryFilesAsync_smoke() { + server.createContext("/a/ok.data", HttpStatusCode.OK, response = "data") + + runBlocking { + val files = JarHttpDownloader.downloadLibraryFilesAsync( + relativePaths = listOf( + JarHttpDownloader.RelativePathToDownload(Path.of("ok.data"), null), + ), + localRepository = localRepository, + remoteRepositories = remoteRepositories, + retry = RetryProvider.disabled(), + downloadDispatcher = Dispatchers.IO, + ) + assertEquals(localRepository.resolve("ok.data"), files.single()) + assertEquals("data", localRepository.resolve("ok.data").readText()) + } + } +} \ No newline at end of file diff --git a/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderTestUtil.kt b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderTestUtil.kt new file mode 100644 index 000000000000..9ac8852ec501 --- /dev/null +++ b/java/idea-ui/testSrc/com/intellij/jarRepository/JarHttpDownloaderTestUtil.kt @@ -0,0 +1,91 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.jarRepository + +import com.intellij.jarRepository.JarRepositoryAuthenticationDataProvider.AuthenticationData +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.call +import io.ktor.server.application.createApplicationPlugin +import io.ktor.server.application.hooks.ResponseSent +import io.ktor.server.application.install +import io.ktor.server.cio.CIO +import io.ktor.server.engine.ApplicationEngine +import io.ktor.server.engine.embeddedServer +import io.ktor.server.request.authorization +import io.ktor.server.request.uri +import io.ktor.server.response.respond +import io.ktor.server.routing.get +import io.ktor.server.routing.routing +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.extension.AfterEachCallback +import org.junit.jupiter.api.extension.BeforeEachCallback +import org.junit.jupiter.api.extension.ExtensionContext +import java.util.* + +object JarHttpDownloaderTestUtil { + private const val LOCALHOST = "127.0.0.1" + + class TestHttpServerExtension(private val init: (ApplicationEngine) -> Unit = {}) : BeforeEachCallback, AfterEachCallback { + lateinit var server: ApplicationEngine + private val logBuffer: StringBuffer = StringBuffer() + val log: String + get() = logBuffer.toString() + + override fun beforeEach(context: ExtensionContext?) { + server = embeddedServer(CIO, host = LOCALHOST, port = 0) { + install(createApplicationPlugin(name = "Log calls") { + on(ResponseSent) { call -> + logBuffer.appendLine("${call.request.uri}: ${call.response.status()?.value}") + } + }) + }.start(wait = false) + init(server) + } + + override fun afterEach(context: ExtensionContext?) { + server.stop(0) + } + } + + fun ApplicationEngine.createContext( + path: String, + httpStatusCode: HttpStatusCode, + log: (String) -> Unit = {}, + response: String? = null, + auth: AuthenticationData? = null, + delayMs: Long = 0, + ) { + application.routing { + get(path) { + delay(delayMs) + + if (auth != null) { + val authString = Base64.getEncoder().encodeToString((auth.userName + ":" + auth.password).toByteArray()) + + val authHeader = call.request.authorization() + if (authHeader == null) { + log("$path: missing auth") + call.respond(HttpStatusCode.Unauthorized) + return@get + } + + if (authHeader != "Basic $authString") { + log("$path: wrong auth") + call.respond(HttpStatusCode.Unauthorized) + return@get + } + } + + log("$path: ${httpStatusCode.value}") + + val bytes = response?.toByteArray() ?: byteArrayOf() + call.respond(httpStatusCode, bytes) + } + } + } + + val ApplicationEngine.url: String + get() = runBlocking { + "http://${resolvedConnectors().first().host}:${resolvedConnectors().first().port}" + } +} \ No newline at end of file diff --git a/java/idea-ui/testSrc/org/jetbrains/idea/maven/utils/library/RepositoryUtilsTest.java b/java/idea-ui/testSrc/org/jetbrains/idea/maven/utils/library/RepositoryUtilsTest.java index dc1dab6fad89..784da8c58cad 100644 --- a/java/idea-ui/testSrc/org/jetbrains/idea/maven/utils/library/RepositoryUtilsTest.java +++ b/java/idea-ui/testSrc/org/jetbrains/idea/maven/utils/library/RepositoryUtilsTest.java @@ -7,7 +7,6 @@ import com.intellij.openapi.roots.OrderRootType; import com.intellij.testFramework.ServiceContainerUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.model.library.JpsMavenRepositoryLibraryDescriptor; -import org.junit.Test; import java.io.IOException; import java.nio.file.Files; @@ -30,7 +29,6 @@ public class RepositoryUtilsTest extends LibraryTest { }); } - @Test public void testLibraryReloadFixesCorruptedJar() throws IOException { var group = "test"; var artifact = "test"; @@ -61,14 +59,12 @@ public class RepositoryUtilsTest extends LibraryTest { assertEquals(corruptedJar, fileContent(jarPath)); // reload library - var result = getResult(RepositoryUtils.deleteAndReloadDependencies(myProject, library)); - assertSize(1, result); + getResult(RepositoryUtils.deleteAndReloadDependencies(myProject, library)); // verify jar became valid assertEquals(validJar, fileContent(jarPath)); } - @Test public void testLibraryReloadDoesNotDeleteUnrelatedFiles() throws IOException { var group = "test"; var artifact = "test"; @@ -100,8 +96,7 @@ public class RepositoryUtilsTest extends LibraryTest { WriteCommandAction.runWriteCommandAction(myProject, () -> modifiableModel.commit()); // reload library - var result = getResult(RepositoryUtils.deleteAndReloadDependencies(myProject, library)); - assertSize(1, result); + getResult(RepositoryUtils.deleteAndReloadDependencies(myProject, library)); // verify file still exists assertTrue(Files.exists(anotherPath)); diff --git a/java/java-impl/resources/META-INF/JavaPlugin.xml b/java/java-impl/resources/META-INF/JavaPlugin.xml index 1260502fa8e6..cf32198f008a 100644 --- a/java/java-impl/resources/META-INF/JavaPlugin.xml +++ b/java/java-impl/resources/META-INF/JavaPlugin.xml @@ -2706,6 +2706,12 @@ + + + +