diff --git a/.idea/modules.xml b/.idea/modules.xml index d4c3b7080c1c..b989420776a5 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -11,6 +11,7 @@ + @@ -182,8 +183,8 @@ - + diff --git a/build/groovy/org/jetbrains/intellij/build/CommunityLibraryLicenses.groovy b/build/groovy/org/jetbrains/intellij/build/CommunityLibraryLicenses.groovy index abcb0e13be1c..c432ad844bc4 100644 --- a/build/groovy/org/jetbrains/intellij/build/CommunityLibraryLicenses.groovy +++ b/build/groovy/org/jetbrains/intellij/build/CommunityLibraryLicenses.groovy @@ -267,6 +267,10 @@ class CommunityLibraryLicenses { license: "Apache 2.0", url: "http://maven.apache.org/", licenseUrl: "http://www.apache.org/licenses/LICENSE-2.0"), new LibraryLicense(name: "plexus-util", version: "2.0.6", license: "Apache 2.0", url: "http://maven.apache.org/", libraryName: 'plexus-utils-2.0.6.jar', licenseUrl: "http://apache.org/licenses/LICENSE-2.0"), + new LibraryLicense(name: "aether", version: "1.1.0", license: "Eclipse Public License 1.0", url: "http://www.eclipse.org/aether/", + libraryName: 'aether-1.1.0-all.jar', licenseUrl: "http://www.eclipse.org/legal/epl-v10.html"), + new LibraryLicense(name: "maven-aether-provider", version: "3.3.9", license: "Apache 2.0", url: "http://maven.apache.org/ref/3.3.9/maven-aether-provider", + libraryName: 'maven-aether-provider-3.3.9-all.jar', licenseUrl: "http://apache.org/licenses/LICENSE-2.0"), new LibraryLicense(name: "Plexus Utils", libraryName: "plexus-utils-1.5.5.jar", version: "1.5.5", license: "Apache 2.0", url: "http://plexus.codehaus.org/plexus-utils"), new LibraryLicense(name: "pockets", attachedTo: "python-helpers", version: "0.2.4", license: "BSD", url: "http://pockets.readthedocs.org/"), diff --git a/build/scripts/layouts.gant b/build/scripts/layouts.gant index d895fe20ec03..80aa006080e6 100644 --- a/build/scripts/layouts.gant +++ b/build/scripts/layouts.gant @@ -15,11 +15,7 @@ */ -import org.jetbrains.intellij.build.BuildContext -import org.jetbrains.intellij.build.BuildOptions -import org.jetbrains.intellij.build.BuildTasks -import org.jetbrains.intellij.build.IdeaCommunityProperties -import org.jetbrains.intellij.build.ProprietaryBuildTools +import org.jetbrains.intellij.build.* import org.jetbrains.jps.util.JpsPathUtil import static org.jetbrains.jps.idea.IdeaProjectLoader.guessHome @@ -340,6 +336,9 @@ public def layoutCommunityPlugins(String home, BuildContext buildContext = null) jar("maven-jps-plugin.jar") { module("maven-jps-plugin") } + jar("aether-dependency-resolver.jar") { + module("aether-dependency-resolver") + } jar("maven-server-api.jar") { module("maven-server-api") } diff --git a/plugins/maven/aether-dependency-resolver/aether-dependency-resolver.iml b/plugins/maven/aether-dependency-resolver/aether-dependency-resolver.iml new file mode 100644 index 000000000000..a0c8f9079d36 --- /dev/null +++ b/plugins/maven/aether-dependency-resolver/aether-dependency-resolver.iml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/plugins/maven/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ArtifactRepositoryManager.java b/plugins/maven/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ArtifactRepositoryManager.java new file mode 100644 index 000000000000..a40e760e11b7 --- /dev/null +++ b/plugins/maven/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ArtifactRepositoryManager.java @@ -0,0 +1,143 @@ +package org.jetbrains.idea.maven.aether; + +import org.apache.maven.repository.internal.MavenRepositorySystemUtils; +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.collection.CollectRequest; +import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.impl.DefaultServiceLocator; +import org.eclipse.aether.repository.LocalRepository; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.resolution.ArtifactResult; +import org.eclipse.aether.resolution.DependencyRequest; +import org.eclipse.aether.resolution.DependencyResult; +import org.eclipse.aether.spi.connector.RepositoryConnectorFactory; +import org.eclipse.aether.spi.connector.transport.TransporterFactory; +import org.eclipse.aether.transfer.TransferEvent; +import org.eclipse.aether.transfer.TransferListener; +import org.eclipse.aether.transport.file.FileTransporterFactory; +import org.eclipse.aether.transport.http.HttpTransporterFactory; +import org.eclipse.aether.util.artifact.JavaScopes; +import org.eclipse.aether.util.filter.DependencyFilterUtils; +import org.eclipse.aether.util.version.GenericVersionScheme; +import org.eclipse.aether.version.InvalidVersionSpecificationException; +import org.eclipse.aether.version.Version; +import org.eclipse.aether.version.VersionScheme; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * @author Eugene Zhuravlev + * Date: 20-Jun-16 + * + * Aether-based repository manager and dependency resolver using maven implementation of this functionality. + * + * instance of this component should be managed by the code which requires dependency resolution functionality + * all necessary params like path to local repo should be passed in constructor + * + */ +public class ArtifactRepositoryManager { + private static final String ARTIFACT_EXTENSION = "jar"; + private final VersionScheme myVersioning = new GenericVersionScheme(); + private final DefaultRepositorySystemSession mySession; + + // todo: more remotes? make remote repos configurable? + public static final List REMOTE_REPOSITORIES = Collections.singletonList( + new RemoteRepository.Builder("central", "default", "http://central.maven.org/maven2/").build() + ); + private static final RepositorySystem ourSystem; + static { + final DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator(); + locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class); + locator.addService(TransporterFactory.class, FileTransporterFactory.class); + locator.addService(TransporterFactory.class, HttpTransporterFactory.class); + locator.setErrorHandler(new DefaultServiceLocator.ErrorHandler() { + public void serviceCreationFailed(Class type, Class impl, Throwable exception) { + if (exception != null) { + throw new RuntimeException(exception); + } + } + }); + ourSystem = locator.getService(RepositorySystem.class); + } + + public ArtifactRepositoryManager(@NotNull File localRepositoryPath) { + this(localRepositoryPath, ProgressConsumer.DEAF); + } + + public ArtifactRepositoryManager(@NotNull File localRepositoryPath, @NotNull final ProgressConsumer progressConsumer) { + final DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession(); + if (progressConsumer != ProgressConsumer.DEAF) { + session.setTransferListener((TransferListener)Proxy + .newProxyInstance(session.getClass().getClassLoader(), new Class[]{TransferListener.class}, new InvocationHandler() { + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + final Object event = args[0]; + if (event instanceof TransferEvent) { + progressConsumer.consume(event.toString()); + //if (((TransferEvent)event).getType() != TransferEvent.EventType.PROGRESSED) { + // progressConsumer.consume(event.toString()); + //} + } + return null; + } + })); + } + // setup session here + + session.setLocalRepositoryManager(ourSystem.newLocalRepositoryManager(session, new LocalRepository(localRepositoryPath))); + session.setReadOnly(); + mySession = session; + } + + public Collection resolveDependency(String groupId, String artifactId, String version) throws Exception { + final DependencyRequest dependencyRequest = new DependencyRequest( + createCollectRequest(groupId, artifactId, toVersion(version)), + DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE) + ); + + final DependencyResult result = ourSystem.resolveDependencies(mySession, dependencyRequest); + final List files = new ArrayList(); + for (ArtifactResult artifactResult : result.getArtifactResults()) { + files.add(artifactResult.getArtifact().getFile()); + } + return files; + } + + private static CollectRequest createCollectRequest(String groupId, String artifactId, Version version) { + return createCollectRequest(groupId, artifactId, Collections.singleton(version)); + } + private static CollectRequest createCollectRequest(String groupId, String artifactId, Collection versions) { + CollectRequest request = new CollectRequest(); + for (Artifact artifact : toArtifacts(groupId, artifactId, versions)) { + request.addDependency(new Dependency(artifact, JavaScopes.COMPILE)); + } + return request.setRepositories(REMOTE_REPOSITORIES); + } + + private Version toVersion(String version) throws InvalidVersionSpecificationException { + return myVersioning.parseVersion(version); + } + + private static List toArtifacts(String groupId, String artifactId, Collection versions) { + if (versions.isEmpty()) { + return Collections.emptyList(); + } + final List result = new ArrayList(versions.size()); + for (Version version : versions) { + result.add(new DefaultArtifact(groupId, artifactId, ARTIFACT_EXTENSION, version.toString())); + } + return result; + } + +} diff --git a/plugins/maven/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ProgressConsumer.java b/plugins/maven/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ProgressConsumer.java new file mode 100644 index 000000000000..c59606bcee2a --- /dev/null +++ b/plugins/maven/aether-dependency-resolver/src/org/jetbrains/idea/maven/aether/ProgressConsumer.java @@ -0,0 +1,14 @@ +package org.jetbrains.idea.maven.aether; + +/** + * @author Eugene Zhuravlev + * Date: 12-Aug-16 + */ +public interface ProgressConsumer { + ProgressConsumer DEAF = new ProgressConsumer() { + public void consume(String message) { + } + }; + + void consume(String message); +} diff --git a/plugins/maven/jps-plugin/maven-jps-plugin.iml b/plugins/maven/jps-plugin/maven-jps-plugin.iml index e64f0c067e9a..236024405ed7 100644 --- a/plugins/maven/jps-plugin/maven-jps-plugin.iml +++ b/plugins/maven/jps-plugin/maven-jps-plugin.iml @@ -23,6 +23,6 @@ + - - + \ No newline at end of file diff --git a/plugins/maven/jps-plugin/src/org/jetbrains/jps/maven/compiler/DependencyResolvingBuilder.java b/plugins/maven/jps-plugin/src/org/jetbrains/jps/maven/compiler/DependencyResolvingBuilder.java new file mode 100644 index 000000000000..e02e3a228270 --- /dev/null +++ b/plugins/maven/jps-plugin/src/org/jetbrains/jps/maven/compiler/DependencyResolvingBuilder.java @@ -0,0 +1,253 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jetbrains.jps.maven.compiler; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.containers.SmartHashSet; +import gnu.trove.THashMap; +import gnu.trove.THashSet; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.idea.maven.aether.ArtifactRepositoryManager; +import org.jetbrains.idea.maven.aether.ProgressConsumer; +import org.jetbrains.jps.ModuleChunk; +import org.jetbrains.jps.api.CanceledStatus; +import org.jetbrains.jps.builders.DirtyFilesHolder; +import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor; +import org.jetbrains.jps.incremental.*; +import org.jetbrains.jps.incremental.messages.BuildMessage; +import org.jetbrains.jps.incremental.messages.CompilerMessage; +import org.jetbrains.jps.incremental.messages.ProgressMessage; +import org.jetbrains.jps.maven.model.RepositoryLibraryDescriptor; +import org.jetbrains.jps.maven.model.impl.JpsMavenRepositoryLibraryType; +import org.jetbrains.jps.model.JpsSimpleElement; +import org.jetbrains.jps.model.library.JpsLibrary; +import org.jetbrains.jps.model.library.JpsOrderRootType; +import org.jetbrains.jps.model.library.JpsTypedLibrary; +import org.jetbrains.jps.model.module.JpsDependencyElement; +import org.jetbrains.jps.model.module.JpsLibraryDependency; +import org.jetbrains.jps.model.module.JpsModule; +import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService; +import org.jetbrains.jps.model.serialization.JpsPathVariablesConfiguration; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Eugene Zhuravlev + * Date: 21-Jun-16 + */ +public class DependencyResolvingBuilder extends ModuleLevelBuilder{ + private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.maven.compiler.DependencyResolvingBuilder"); + private static final String NAME = "Maven Dependency Resolver"; + private static final String MAVEN_REPOSITORY_PATH_VAR = "MAVEN_REPOSITORY"; + private static final String DEFAULT_MAVEN_REPOSITORY_PATH = ".m2/repository"; + + private static final Key MANAGER_KEY = Key.create("_artifact_repository_manager_"); + private static final Key RESOLVE_ERROR_KEY = Key.create("_artifact_repository_resolve_error_"); + + public DependencyResolvingBuilder() { + super(BuilderCategory.INITIAL); + } + + @NotNull + @Override + public String getPresentableName() { + return NAME; + } + + public void buildStarted(CompileContext context) { + ResourceGuard.init(context); + } + + @Override + public void chunkBuildStarted(CompileContext context, ModuleChunk chunk) { + try { + checkDependencies(context, chunk); + } + catch (Exception e) { + context.putUserData(RESOLVE_ERROR_KEY, e); + } + } + + @Override + public ExitCode build(CompileContext context, + ModuleChunk chunk, + DirtyFilesHolder dirtyFilesHolder, + OutputConsumer outputConsumer) throws ProjectBuildException, IOException { + + final Exception error = context.getUserData(RESOLVE_ERROR_KEY); + if (error != null) { + final StringBuilder builder = new StringBuilder().append("Error resolving dependencies for ").append(chunk.getPresentableShortName()); + final String details = error.getMessage(); + if (details != null) { + builder.append(": ").append(details); + } + final String msg = builder.toString(); + LOG.info(msg, error); + context.processMessage(new CompilerMessage(NAME, BuildMessage.Kind.ERROR, msg)); + return ExitCode.ABORT; + } + + return ExitCode.OK; + } + + private static void checkDependencies(CompileContext context, ModuleChunk chunk) throws Exception { + final Collection>> libs = getRepositoryLibraries(chunk); + if (!libs.isEmpty()) { + final ArtifactRepositoryManager repoManager = getRepositoryManager(context); + for (JpsTypedLibrary> lib : libs) { + final RepositoryLibraryDescriptor descriptor = lib.getProperties().getData(); + final ResourceGuard guard = ResourceGuard.get(context, descriptor); + if (guard.requestProcessing(context.getCancelStatus())) { + try { + final Collection resolved = repoManager.resolveDependency(descriptor.getGroupId(), descriptor.getArtifactId(), descriptor.getVersion()); + syncPaths(resolved, lib); + } + finally { + guard.finish(); + } + } + } + } + } + + private static void syncPaths(@NotNull Collection resolved, @NotNull JpsTypedLibrary> lib) throws Exception { + if (resolved.isEmpty()) { + LOG.info("No artifacts were resolved for repository dependency " + lib.getProperties().getData().getMavenId()); + return; + } + final THashSet libFiles = new THashSet(FileUtil.FILE_HASHING_STRATEGY); + libFiles.addAll(lib.getFiles(JpsOrderRootType.COMPILED)); + libFiles.removeAll(resolved); + + if (!libFiles.isEmpty()) { + Map nameToArtifactMap = null; + for (File file : libFiles) { + if (file.exists()) { + continue; + } + + if (nameToArtifactMap == null) { // lazy init + nameToArtifactMap = new THashMap(FileUtil.PATH_HASHING_STRATEGY); + for (File f : resolved) { + final File prev = nameToArtifactMap.put(f.getName(), f); + if (prev != null) { + throw new Exception("Ambiguous artifacts with the same name: " + prev.getPath() + " and " + f.getPath()); + } + } + } + + final File resolvedArtifact = nameToArtifactMap.get(file.getName()); + if (resolvedArtifact != null) { + FileUtil.copy(resolvedArtifact, file); + } + } + } + } + + private static final class ResourceGuard { + private static final Key> CONTEXT_KEY = GlobalContextKey.create("_artifact_repository_resolved_libraries_"); + private static final byte INITIAL = 0; + private static final byte PROGRESS = 1; + private static final byte FINISHED = 2; + private byte myState = INITIAL; + + synchronized boolean requestProcessing(final CanceledStatus cancelStatus) { + if (myState == INITIAL) { + myState = PROGRESS; + return true; + } + while (myState == PROGRESS) { + if (cancelStatus.isCanceled()) { + break; + } + try { + this.wait(1000L); + } + catch (InterruptedException ignored) { + } + } + return false; + } + + synchronized void finish() { + myState = FINISHED; + this.notifyAll(); + } + + static void init(CompileContext context) { + context.putUserData(CONTEXT_KEY, ContainerUtil.newConcurrentMap()); + } + + @NotNull + static ResourceGuard get(CompileContext context, RepositoryLibraryDescriptor descriptor) { + final ConcurrentMap map = context.getUserData(CONTEXT_KEY); + assert map != null; + final ResourceGuard g = new ResourceGuard(); + final ResourceGuard existing = map.putIfAbsent(descriptor, g); + return existing != null? existing : g; + } + } + + @NotNull + private static Collection>> getRepositoryLibraries(ModuleChunk chunk) { + final Collection>> result = new SmartHashSet>>(); + for (JpsModule module : chunk.getModules()) { + for (JpsDependencyElement dep : module.getDependenciesList().getDependencies()) { + if (dep instanceof JpsLibraryDependency) { + final JpsLibrary _lib = ((JpsLibraryDependency)dep).getLibrary(); + final JpsTypedLibrary> lib = _lib != null ? _lib.asTyped(JpsMavenRepositoryLibraryType.INSTANCE) : null; + if (lib != null) { + result.add(lib); + } + } + } + } + return result; + } + + private static ArtifactRepositoryManager getRepositoryManager(final CompileContext context) { + ArtifactRepositoryManager manager = MANAGER_KEY.get(context); + if (manager == null) { + manager = new ArtifactRepositoryManager(getLocalRepoDir(context), new ProgressConsumer() { + public void consume(String message) { + context.processMessage(new ProgressMessage(message)); + } + }); + // further init manager here + MANAGER_KEY.set(context, manager); + } + return manager; + } + + @NotNull + private static File getLocalRepoDir(CompileContext context) { + final JpsPathVariablesConfiguration pvConfig = JpsModelSerializationDataService.getPathVariablesConfiguration(context.getProjectDescriptor().getModel().getGlobal()); + final String localRepoPath = pvConfig != null? pvConfig.getUserVariableValue(MAVEN_REPOSITORY_PATH_VAR) : null; + if (localRepoPath != null) { + return new File(localRepoPath); + } + final String root = System.getProperty("user.home", null); + return root != null ? new File(root, DEFAULT_MAVEN_REPOSITORY_PATH) : new File(DEFAULT_MAVEN_REPOSITORY_PATH); + } + +} diff --git a/plugins/maven/jps-plugin/src/org/jetbrains/jps/maven/compiler/MavenBuilderService.java b/plugins/maven/jps-plugin/src/org/jetbrains/jps/maven/compiler/MavenBuilderService.java index dcfca72a5432..dc2b5c6fe65a 100644 --- a/plugins/maven/jps-plugin/src/org/jetbrains/jps/maven/compiler/MavenBuilderService.java +++ b/plugins/maven/jps-plugin/src/org/jetbrains/jps/maven/compiler/MavenBuilderService.java @@ -18,6 +18,7 @@ package org.jetbrains.jps.maven.compiler; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.BuildTargetType; import org.jetbrains.jps.incremental.BuilderService; +import org.jetbrains.jps.incremental.ModuleLevelBuilder; import org.jetbrains.jps.incremental.TargetBuilder; import org.jetbrains.jps.maven.model.impl.MavenResourcesTargetType; @@ -41,4 +42,9 @@ public class MavenBuilderService extends BuilderService{ public List> createBuilders() { return Collections.singletonList(new MavenResourcesBuilder()); } + + @NotNull + public List createModuleLevelBuilders() { + return Collections.singletonList(new DependencyResolvingBuilder()); + } } diff --git a/plugins/maven/lib/aether-1.1.0-all.jar b/plugins/maven/lib/aether-1.1.0-all.jar new file mode 100644 index 000000000000..9d74aa48759c Binary files /dev/null and b/plugins/maven/lib/aether-1.1.0-all.jar differ diff --git a/plugins/maven/lib/maven-aether-provider-3.3.9-all.jar b/plugins/maven/lib/maven-aether-provider-3.3.9-all.jar new file mode 100644 index 000000000000..4eaa093d8d6a Binary files /dev/null and b/plugins/maven/lib/maven-aether-provider-3.3.9-all.jar differ diff --git a/plugins/maven/src/main/resources/META-INF/plugin.xml b/plugins/maven/src/main/resources/META-INF/plugin.xml index a2c622103b73..346425502e7c 100644 --- a/plugins/maven/src/main/resources/META-INF/plugin.xml +++ b/plugins/maven/src/main/resources/META-INF/plugin.xml @@ -71,7 +71,7 @@ - +