diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java index 3e4e1372759c..9357973d6549 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java @@ -3,11 +3,14 @@ package com.intellij.execution.testDiscovery; import com.intellij.openapi.diagnostic.Logger; import com.intellij.rt.coverage.data.api.TestDiscoveryProtocolReader; +import com.intellij.util.ArrayUtil; import com.intellij.util.containers.MultiMap; import gnu.trove.TIntObjectHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.Arrays; + class IdeaTestDiscoveryProtocolReader implements TestDiscoveryProtocolReader, TestDiscoveryProtocolReader.NameEnumeratorReader { private static final Logger LOG = Logger.getInstance(IdeaTestDiscoveryProtocolReader.class); @@ -87,6 +90,7 @@ class IdeaTestDiscoveryProtocolReader implements TestDiscoveryProtocolReader, Te private final String myTestClassName = myTestExecutionNameEnumerator.get(testClassId); private final String myTestMethodName = myTestExecutionNameEnumerator.get(testMethodId); private final MultiMap myUsedMethods = new MultiMap<>(); + private String[] myUsedFiles = ArrayUtil.EMPTY_STRING_ARRAY; private int myCurrentClassId; @Override @@ -112,12 +116,12 @@ class IdeaTestDiscoveryProtocolReader implements TestDiscoveryProtocolReader, Te @Override public void testDataProcessed() { - myIndex.updateTestData(myTestClassName, myTestMethodName, myUsedMethods, myModuleName, myFrameworkId); + myIndex.updateTestData(myTestClassName, myTestMethodName, myUsedMethods, myUsedFiles, myModuleName, myFrameworkId); } @Override public void processAffectedFile(int[] ints) { - // todo[batkovich] update index + myUsedFiles = Arrays.stream(ints).mapToObj(id -> myTestExecutionNameEnumerator.get(id)).toArray(ArrayUtil.STRING_ARRAY_FACTORY::create); } }; } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/IntellijTestDiscoveryProducer.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/IntellijTestDiscoveryProducer.java index 530b140f520c..2ea074f686cf 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/IntellijTestDiscoveryProducer.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/IntellijTestDiscoveryProducer.java @@ -27,8 +27,7 @@ public class IntellijTestDiscoveryProducer implements TestDiscoveryProducer { @Override public MultiMap getDiscoveredTests(@NotNull Project project, @NotNull List> classesAndMethods, - byte frameworkId, - @NotNull List filePaths) { + byte frameworkId) { if (!ApplicationManager.getApplication().isInternal()) { return MultiMap.empty(); } @@ -76,7 +75,7 @@ public class IntellijTestDiscoveryProducer implements TestDiscoveryProducer { @NotNull @Override - public MultiMap getDiscoveredTests(@NotNull Project project, @NotNull List filePaths) { + public MultiMap getDiscoveredTestsForFiles(@NotNull Project project, @NotNull List filePaths, byte frameworkId) { try { return request(filePaths, s -> "\"" + s + "\"", "files"); } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java index 6a350cda434f..aa225235c374 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java @@ -6,7 +6,7 @@ import com.intellij.openapi.util.Couple; import com.intellij.util.containers.MultiMap; import org.jetbrains.annotations.NotNull; -import java.util.Collections; +import java.io.IOException; import java.util.List; public class LocalTestDiscoveryProducer implements TestDiscoveryProducer { @@ -14,8 +14,7 @@ public class LocalTestDiscoveryProducer implements TestDiscoveryProducer { @NotNull public MultiMap getDiscoveredTests(@NotNull Project project, @NotNull List> classesAndMethods, - byte frameworkId, - @NotNull List filePaths) { + byte frameworkId) { MultiMap result = new MultiMap<>(); TestDiscoveryIndex instance = TestDiscoveryIndex.getInstance(project); classesAndMethods.forEach(couple -> result.putAllValues(couple.second == null ? @@ -24,20 +23,25 @@ public class LocalTestDiscoveryProducer implements TestDiscoveryProducer { return result; } + @NotNull + @Override + public MultiMap getDiscoveredTestsForFiles(@NotNull Project project, @NotNull List paths, byte frameworkId) { + MultiMap result = new MultiMap<>(); + TestDiscoveryIndex instance = TestDiscoveryIndex.getInstance(project); + for (String path : paths) { + result.putAllValues(instance.getTestsByFile(path, frameworkId)); + } + return result; + } + + @NotNull + @Override + public List getAffectedFilePaths(@NotNull Project project, @NotNull List testFqns) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public boolean isRemote() { return false; } - - @NotNull - @Override - public MultiMap getDiscoveredTests(@NotNull Project project, @NotNull List filePaths) { - return MultiMap.empty(); - } - - @NotNull - @Override - public List getAffectedFilePaths(@NotNull Project project, @NotNull List testFqns) { - return Collections.emptyList(); - } } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryExtension.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryExtension.java index 3ae1ed9bd497..d6f1a94ff8aa 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryExtension.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryExtension.java @@ -85,6 +85,7 @@ public class TestDiscoveryExtension extends RunConfigurationExtension { if (agentPath == null) return; params.getVMParametersList().add("-javaagent:" + agentPath); TestDiscoveryDataSocketListener listener = tryInstallSocketListener(configuration); + params.getVMParametersList().addProperty(SocketTestDiscoveryProtocolDataListener.DATA_VERSION, String.valueOf(3)); if (listener != null) { params.getVMParametersList().addProperty(SocketTestDiscoveryProtocolDataListener.PORT_PROP, Integer.toString(listener.getPort())); params.getVMParametersList().addProperty(SocketTestDiscoveryProtocolDataListener.HOST_PROP, "127.0.0.1"); diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java index bca465290fc2..5bb985357d09 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java @@ -56,6 +56,13 @@ public class TestDiscoveryIndex implements Disposable { }); } + @NotNull + public MultiMap getTestsByFile(String relativePath, byte frameworkId) { + MultiMap map = executeUnderLock(holder -> holder.getTestsByFile(relativePath, frameworkId)); + return map == null ? MultiMap.empty() : map; + } + + @NotNull public MultiMap getTestsByClassName(@NotNull String classFQName, byte frameworkId) { MultiMap map = executeUnderLock(holder -> holder.getTestsByClassName(classFQName, frameworkId)); @@ -88,10 +95,11 @@ public class TestDiscoveryIndex implements Disposable { public void updateTestData(@NotNull String testClassName, @NotNull String testMethodName, @NotNull MultiMap usedMethods, + @NotNull String[] usedFiles, @Nullable String moduleName, byte frameworkId) { executeUnderLock(holder -> { - holder.updateTestData(testClassName, testMethodName, usedMethods, moduleName, frameworkId); + holder.updateTestData(testClassName, testMethodName, usedMethods, usedFiles, moduleName, frameworkId); return null; }); } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryProducer.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryProducer.java index 2c2054032d1f..514e179913a6 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryProducer.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryProducer.java @@ -29,13 +29,14 @@ public interface TestDiscoveryProducer { @NotNull MultiMap getDiscoveredTests(@NotNull Project project, @NotNull List> classesAndMethods, - byte frameworkId, - @NotNull List filePaths); - - boolean isRemote(); + byte frameworkId); @NotNull - MultiMap getDiscoveredTests(@NotNull Project project, @NotNull List filePaths); + MultiMap getDiscoveredTestsForFiles(@NotNull Project project, + @NotNull List paths, + byte frameworkId); + + boolean isRemote(); static void consumeDiscoveredTests(@NotNull Project project, @NotNull List> classesAndMethods, @@ -51,8 +52,8 @@ public interface TestDiscoveryProducer { }; for (TestDiscoveryProducer producer : EP.getExtensions()) { for (Map.Entry> entry : ContainerUtil.concat( - producer.getDiscoveredTests(project, classesAndMethods, frameworkId, filePaths).entrySet(), - producer.getDiscoveredTests(project, filePaths).entrySet())) { + producer.getDiscoveredTests(project, classesAndMethods, frameworkId).entrySet(), + producer.getDiscoveredTestsForFiles(project, filePaths, frameworkId).entrySet())) { String className = entry.getKey(); for (String methodRawName : entry.getValue()) { if (!visitedTests.get(className).contains(methodRawName)) { @@ -68,6 +69,7 @@ public interface TestDiscoveryProducer { @NotNull List getAffectedFilePaths(@NotNull Project project, @NotNull List testFqns) throws IOException; + //TODO use it [Dmitry Batkovich] static void consumeAffectedPaths(@NotNull Project project, @NotNull List testFqns, @NotNull Consumer pathsConsumer) throws IOException { for (TestDiscoveryProducer extension : EP.getExtensions()) { for (String path : extension.getAffectedFilePaths(project, testFqns)) { diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java index e6fcd166de70..4dd4bb48b44d 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java @@ -27,25 +27,29 @@ public final class DiscoveredTestDataHolder { private static final Logger LOG = Logger.getInstance(DiscoveredTestDataHolder.class); private final DiscoveredTestsIndex myDiscoveredTestsIndex; + private final TestFilesIndex myTestFilesIndex; private final TestModuleIndex myTestModuleIndex; private final PersistentStringEnumerator myClassEnumerator; private final PersistentStringEnumerator myMethodEnumerator; + private final PersistentStringEnumerator myPathEnumerator; private final PersistentEnumeratorDelegate myTestEnumerator; private final PersistentObjectSeq myConstructedDataFiles = new PersistentObjectSeq(); private boolean myDisposed; private final Disposable myDisposable = Disposer.newDisposable(); - static final int VERSION = 9; + static final int VERSION = 10; public DiscoveredTestDataHolder(@NotNull Path basePath) { final Path versionFile = getVersionFile(basePath); PathKt.createDirectories(basePath); final File discoveredTestsIndexFile = basePath.resolve("discoveredTests.index").toFile(); + final File testFilesIndexFile = basePath.resolve("testFiles.index").toFile(); final File classNameEnumeratorFile = basePath.resolve("className.enum").toFile(); final File methodNameEnumeratorFile = basePath.resolve("methodName.enum").toFile(); + final File pathEnumeratorFile = basePath.resolve("path.enum").toFile(); final File testNameEnumeratorFile = basePath.resolve("testName.enum").toFile(); try { @@ -59,9 +63,11 @@ public final class DiscoveredTestDataHolder { } DiscoveredTestsIndex discoveredTestsIndex; + TestFilesIndex testFilesIndex; TestModuleIndex testModuleIndex; PersistentStringEnumerator classNameEnumerator; PersistentStringEnumerator methodEnumerator; + PersistentStringEnumerator pathEnumerator; PersistentEnumeratorDelegate testEnumerator; int iterations = 0; @@ -73,6 +79,9 @@ public final class DiscoveredTestDataHolder { discoveredTestsIndex = new DiscoveredTestsIndex(discoveredTestsIndexFile); myConstructedDataFiles.add(discoveredTestsIndex); + testFilesIndex = new TestFilesIndex(testFilesIndexFile); + myConstructedDataFiles.add(testFilesIndex); + testModuleIndex = new TestModuleIndex(basePath, myConstructedDataFiles); classNameEnumerator = new PersistentStringEnumerator(classNameEnumeratorFile, true); @@ -81,6 +90,9 @@ public final class DiscoveredTestDataHolder { methodEnumerator = new PersistentStringEnumerator(methodNameEnumeratorFile, true); myConstructedDataFiles.add(methodEnumerator); + pathEnumerator = new PersistentStringEnumerator(pathEnumeratorFile, true); + myConstructedDataFiles.add(pathEnumerator); + testEnumerator = new PersistentEnumeratorDelegate<>(testNameEnumeratorFile, TestId.DESCRIPTOR, 1024 * 4); myConstructedDataFiles.add(testEnumerator); @@ -102,9 +114,11 @@ public final class DiscoveredTestDataHolder { } myDiscoveredTestsIndex = discoveredTestsIndex; + myTestFilesIndex = testFilesIndex; myTestModuleIndex = testModuleIndex; myClassEnumerator = classNameEnumerator; myMethodEnumerator = methodEnumerator; + myPathEnumerator = pathEnumerator; myTestEnumerator = testEnumerator; LowMemoryWatcher.register(() -> myConstructedDataFiles.flush(), myDisposable); @@ -114,7 +128,6 @@ public final class DiscoveredTestDataHolder { } } - private static void writeVersion(@NotNull Path versionFile) throws IOException { try (final DataOutputStream versionOut = new DataOutputStream(PathKt.outputStream(versionFile))) { DataInputOutputUtil.writeINT(versionOut, VERSION); @@ -170,6 +183,7 @@ public final class DiscoveredTestDataHolder { public void updateTestData(@NotNull String testClassName, @NotNull String testMethodName, @NotNull MultiMap usedMethods, + @NotNull String[] usedFiles, @Nullable String moduleName, byte frameworkId) throws IOException { @@ -182,10 +196,35 @@ public final class DiscoveredTestDataHolder { methodIds.add(myMethodEnumerator.enumerate(methodName)); } } - myDiscoveredTestsIndex.update(testNameId, new DiscoveredTestsIndex.UsedMethods(result)).compute(); + + Map usedVirtualFileIds = new HashMap<>(); + for (String file : usedFiles) { + int fileId = myPathEnumerator.enumerate(file); + usedVirtualFileIds.put(fileId, null); + } + + UsedSources usedSources = new UsedSources(result, usedVirtualFileIds); + myDiscoveredTestsIndex.update(testNameId, usedSources).compute(); + myTestFilesIndex.update(testNameId, usedSources).compute(); myTestModuleIndex.appendModuleData(testNameId, moduleName); } + @NotNull + public MultiMap getTestsByFile(@NotNull String relativePath, byte frameworkId) throws IOException { + int fileId = myPathEnumerator.tryEnumerate(relativePath); + if (fileId == 0) return MultiMap.empty(); + try { + MultiMap result = new MultiMap<>(); + IOException[] exception = {null}; + myTestFilesIndex.getData(fileId).forEach((testId, v) -> consumeDiscoveredTest(testId, frameworkId, result, exception)); + if (exception[0] != null) throw exception[0]; + return result; + } + catch (StorageException e) { + throw new IOException(e); + } + } + @NotNull public MultiMap getTestsByClassName(@NotNull String classFQName, byte frameworkId) throws IOException { int classId = myClassEnumerator.tryEnumerate(classFQName); diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java index a6dca04bcbce..c644bff2ea1e 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java @@ -15,9 +15,8 @@ import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; import java.util.Collection; -import java.util.Map; -public class DiscoveredTestsIndex extends MapReduceIndex { +public class DiscoveredTestsIndex extends MapReduceIndex { protected DiscoveredTestsIndex(@NotNull File file) throws IOException { super(INDEX_EXTENSION, new MyIndexStorage(file), new MyForwardIndex() { @NotNull @@ -54,7 +53,7 @@ public class DiscoveredTestsIndex extends MapReduceIndex INDEX_EXTENSION = new IndexExtension() { + private static final IndexExtension INDEX_EXTENSION = new IndexExtension() { @NotNull @Override public IndexId getName() { @@ -63,7 +62,7 @@ public class DiscoveredTestsIndex extends MapReduceIndex getIndexer() {return inputData -> inputData.myTestUsedMethods;} + public DataIndexer getIndexer() {return inputData -> inputData.myUsedMethods;} @NotNull @Override @@ -93,9 +92,4 @@ public class DiscoveredTestsIndex extends MapReduceIndex myTestUsedMethods; - UsedMethods(Map methods) {myTestUsedMethods = methods;} - } } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestFilesIndex.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestFilesIndex.java new file mode 100644 index 000000000000..fc9cb3888292 --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestFilesIndex.java @@ -0,0 +1,95 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.util.indexing.DataIndexer; +import com.intellij.util.indexing.IndexExtension; +import com.intellij.util.indexing.IndexId; +import com.intellij.util.indexing.impl.*; +import com.intellij.util.io.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +public class TestFilesIndex extends MapReduceIndex { + protected TestFilesIndex(@NotNull File file) throws IOException { + super(INDEX_EXTENSION, new MyIndexStorage(file), new MyForwardIndex() { + @NotNull + @Override + public PersistentHashMap> createMap() throws IOException { + return new PersistentHashMap<>(new File(file, "forward.idx"), EnumeratorIntegerDescriptor.INSTANCE, + new IntCollectionDataExternalizer()); + } + }); + } + + @Override + public void checkCanceled() { + ProgressManager.checkCanceled(); + } + + @Override + protected void requestRebuild(@NotNull Throwable e) { + //TODO index corrupted + } + + @Nullable + Collection getTestDataFor(int testId) throws IOException { + return ((MyForwardIndex)myForwardIndex).containsDataFrom(testId); + } + + private static class MyIndexStorage extends MapIndexStorage { + protected MyIndexStorage(@NotNull File storageFile) throws IOException { + super(storageFile, EnumeratorIntegerDescriptor.INSTANCE, VoidDataExternalizer.INSTANCE, 4 * 1024, false); + } + + @Override + protected void checkCanceled() { + ProgressManager.checkCanceled(); + } + } + + private static final IndexExtension INDEX_EXTENSION = new IndexExtension() { + @NotNull + @Override + public IndexId getName() { + return IndexId.create("jvm.discovered.test.files"); + } + + @NotNull + @Override + public DataIndexer getIndexer() {return inputData -> inputData.myUsedFiles;} + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return EnumeratorIntegerDescriptor.INSTANCE; + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return VoidDataExternalizer.INSTANCE; + } + + @Override + public int getVersion() { + return DiscoveredTestDataHolder.VERSION; + } + }; + + + private abstract static class MyForwardIndex extends KeyCollectionBasedForwardIndex { + protected MyForwardIndex() throws IOException { + super(INDEX_EXTENSION); + } + + @Nullable + public Collection containsDataFrom(int testId) throws IOException { + return getInput(testId); + } + } +} diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/UsedSources.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/UsedSources.java new file mode 100644 index 000000000000..c3546b9d2dbc --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/UsedSources.java @@ -0,0 +1,18 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import gnu.trove.TIntArrayList; +import org.jetbrains.annotations.NotNull; + +import java.util.Map; + +class UsedSources { + @NotNull + final Map myUsedMethods; + @NotNull + final Map myUsedFiles; + UsedSources(@NotNull Map methods, @NotNull Map files) { + myUsedMethods = methods; + myUsedFiles = files; + } +}