From 0ab773efd82d21fd053eea7a48ed6c082d52473c Mon Sep 17 00:00:00 2001 From: Dmitry Batkovich Date: Tue, 6 Feb 2018 16:36:03 +0300 Subject: [PATCH] test discovery: rework local indices --- .../IdeaTestDiscoveryProtocolReader.java | 11 +- .../LocalTestDiscoveryProducer.java | 8 +- .../testDiscovery/TestDiscoveryIndex.java | 133 +---- .../TestDiscoverySearchHelper.java | 11 +- .../testDiscovery/TestInfoHolder.java | 557 ------------------ .../indices/DiscoveredTestDataHolder.java | 237 ++++++++ .../indices/DiscoveredTestsIndex.java | 228 +++++++ .../indices/MethodQNameSerializer.java | 35 ++ .../indices/PersistentObjectSeq.java | 81 +++ .../testDiscovery/indices/TestId.java | 74 +++ .../indices/TestModuleIndex.java | 80 +++ .../TestDiscoveryJUnitIntegrationTest.java | 11 +- 12 files changed, 779 insertions(+), 687 deletions(-) delete mode 100644 java/execution/impl/src/com/intellij/execution/testDiscovery/TestInfoHolder.java create mode 100644 java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java create mode 100644 java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java create mode 100644 java/execution/impl/src/com/intellij/execution/testDiscovery/indices/MethodQNameSerializer.java create mode 100644 java/execution/impl/src/com/intellij/execution/testDiscovery/indices/PersistentObjectSeq.java create mode 100644 java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestId.java create mode 100644 java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestModuleIndex.java diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java index 90b04b812913..40efd5d2f96d 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/IdeaTestDiscoveryProtocolReader.java @@ -8,8 +8,6 @@ import gnu.trove.TIntObjectHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.IOException; - class IdeaTestDiscoveryProtocolReader implements TestDiscoveryProtocolReader, TestDiscoveryProtocolReader.NameEnumeratorReader { private static final Logger LOG = Logger.getInstance(IdeaTestDiscoveryProtocolReader.class); @@ -42,7 +40,7 @@ class IdeaTestDiscoveryProtocolReader implements TestDiscoveryProtocolReader, Te public MetadataReader createMetadataReader() { return new MetadataReader() { @Override - public void processMetadataEntry(String s, String s1) { + public void processMetadataEntry(String k, String v) { // do nothing } }; @@ -85,12 +83,7 @@ class IdeaTestDiscoveryProtocolReader implements TestDiscoveryProtocolReader, Te @Override public void testDataProcessed() { - try { - myIndex.updateFromData(myTestClassName, myTestMethodName, myUsedMethods, myModuleName, myFrameworkId); - } - catch (IOException e) { - LOG.error(e); - } + myIndex.updateTestData(myTestClassName, myTestMethodName, myUsedMethods, myModuleName, myFrameworkId); } }; } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java index 09d5f5f2bb6b..8a76c8d28b88 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/LocalTestDiscoveryProducer.java @@ -19,13 +19,7 @@ public class LocalTestDiscoveryProducer implements TestDiscoveryProducer { @NotNull String classFQName, @NotNull String methodName, byte frameworkId) { - try { - return TestDiscoveryIndex.getInstance(project).getTestsByMethodName(classFQName, methodName, frameworkId); - } - catch (IOException e) { - TestDiscoveryProducer.LOG.error(e); - return MultiMap.empty(); - } + return TestDiscoveryIndex.getInstance(project).getTestsByMethodName(classFQName, methodName, frameworkId); } @Override diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java index c9ce4229ce15..4331da15935f 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java @@ -1,6 +1,7 @@ // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.execution.testDiscovery; +import com.intellij.execution.testDiscovery.indices.DiscoveredTestDataHolder; import com.intellij.openapi.Disposable; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; @@ -9,8 +10,6 @@ import com.intellij.openapi.startup.StartupManager; import com.intellij.util.ThrowableConvertor; import com.intellij.util.containers.MultiMap; import com.intellij.util.io.PathKt; -import gnu.trove.TIntArrayList; -import gnu.trove.TIntObjectHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -19,13 +18,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -/** - * @author Maxim.Mossienko on 7/9/2015. - */ public class TestDiscoveryIndex implements Disposable { static final Logger LOG = Logger.getInstance(TestDiscoveryIndex.class); - private volatile TestInfoHolder myHolder; + private volatile DiscoveredTestDataHolder myHolder; private final Object myLock = new Object(); private final Path myBasePath; @@ -47,94 +43,34 @@ public class TestDiscoveryIndex implements Disposable { } } - public boolean hasTestTrace(@NotNull String testClassName, @NotNull String testMethodName, byte frameworkId) throws IOException { - Boolean result = withTestDataHolder(localHolder -> { - TestInfoHolder.TestId testId = localHolder.createTestId(testClassName, testMethodName, frameworkId); - final int testNameId = localHolder.myTestEnumerator.tryEnumerate(testId); - return testNameId != 0 && localHolder.myTestNameToUsedClassesAndMethodMap.get(testNameId) != null; - }); + public boolean hasTestTrace(@NotNull String testClassName, @NotNull String testMethodName, byte frameworkId) { + Boolean result = executeUnderLock(holder -> holder.hasTestTrace(testClassName, testMethodName, frameworkId)); return result == Boolean.TRUE; } - public void removeTestTrace(@NotNull String testClassName, @NotNull String testMethodName, byte frameworkId) throws IOException { - withTestDataHolder(localHolder -> { - TestInfoHolder.TestId testId = localHolder.createTestId(testClassName, testMethodName, frameworkId); - final int testNameId = localHolder.myTestEnumerator.tryEnumerate(testId); - if (testNameId != 0) { - localHolder.doUpdateFromDiff(testNameId, null, - localHolder.myTestNameToUsedClassesAndMethodMap.get(testNameId), - null); - } + public void removeTestTrace(@NotNull String testClassName, @NotNull String testMethodName, byte frameworkId) { + executeUnderLock(holder -> { + holder.removeTestTrace(testClassName, testMethodName, frameworkId); return null; }); } @NotNull - public MultiMap getTestsByMethodName(@NotNull String classFQName, @NotNull String methodName, byte frameworkId) throws IOException { - MultiMap map = withTestDataHolder(new ThrowableConvertor, IOException>() { - @Override - public MultiMap convert(TestInfoHolder localHolder) throws IOException { - Collection ids = getTestIdsByMethod(localHolder); - if (ids.isEmpty()) return MultiMap.empty(); - - MultiMap result = new MultiMap<>(); - for (TestInfoHolder.TestId id : ids) { - if (id.getFrameworkId() == frameworkId) { - result.putValue(localHolder.myClassEnumerator.valueOf(id.getClassId()), - localHolder.myMethodEnumerator.valueOf(id.getMethodId())); - } - } - return result; - } - - private Collection getTestIdsByMethod(TestInfoHolder localHolder) throws IOException { - final TIntArrayList localList = localHolder.myMethodQNameToTestNames.get( - TestInfoHolder.createKey( - localHolder.myClassEnumerator.enumerate(classFQName), - localHolder.myMethodEnumerator.enumerate(methodName) - ) - ); - if (localList == null) return Collections.emptyList(); - - final ArrayList result = new ArrayList<>(localList.size()); - for (int testNameId : localList.toNativeArray()) { - if (testNameId < 0) { - continue; - } - result.add(localHolder.myTestEnumerator.valueOf(testNameId)); - } - return result; - } - }); + public MultiMap getTestsByMethodName(@NotNull String classFQName, @NotNull String methodName, byte frameworkId) { + MultiMap map = executeUnderLock(holder -> holder.getTestsByMethodName(classFQName, methodName, frameworkId)); return map == null ? MultiMap.empty() : map; } @NotNull - public Collection getTestModulesByMethodName(@NotNull String classFQName, @NotNull String methodName, byte frameworkId) throws IOException { - List modules = withTestDataHolder(localHolder -> { - final TIntArrayList list = localHolder.myTestNameToNearestModule.get( - TestInfoHolder.createKey( - localHolder.myClassEnumerator.enumerate(classFQName), - localHolder.myMethodEnumerator.enumerate(methodName) - ) - ); - if (list == null) return Collections.emptyList(); - final ArrayList result = new ArrayList<>(list.size()); - for (int moduleNameId : list.toNativeArray()) { - final TestInfoHolder.ModuleId moduleNameWithPrefix = localHolder.myModuleEnumerator.valueOf(moduleNameId); - if (moduleNameWithPrefix != null && moduleNameWithPrefix.getFrameworkId() == frameworkId) { - result.add(moduleNameWithPrefix.getModuleName()); - } - } - return result; - }); + public Collection getTestModulesByMethodName(@NotNull String classFQName, @NotNull String methodName, byte frameworkId) { + Collection modules = executeUnderLock(holder -> holder.getTestModulesByMethodName(classFQName, methodName, frameworkId)); return modules == null ? Collections.emptySet() : modules; } @Override public void dispose() { synchronized (myLock) { - TestInfoHolder holder = myHolder; + DiscoveredTestDataHolder holder = myHolder; if (holder != null) { holder.dispose(); myHolder = null; @@ -142,58 +78,43 @@ public class TestDiscoveryIndex implements Disposable { } } - public void updateFromData(@NotNull String testClassName, + public void updateTestData(@NotNull String testClassName, @NotNull String testMethodName, @NotNull MultiMap usedMethods, @Nullable String moduleName, - byte frameworkId) throws IOException { - withTestDataHolder(localHolder -> { - final int testNameId = localHolder.myTestEnumerator.enumerate(localHolder.createTestId(testClassName, testMethodName, frameworkId)); - TIntObjectHashMap result = new TIntObjectHashMap<>(); - for (Map.Entry> e : usedMethods.entrySet()) { - int classId = localHolder.myClassEnumerator.enumerate(e.getKey()); - TIntArrayList methodIds = new TIntArrayList(); - result.put(classId, methodIds); - for (String methodName : e.getValue()) { - methodIds.add(localHolder.myMethodEnumerator.enumerate(methodName)); - } - } - TIntObjectHashMap previousClassData = localHolder.myTestNameToUsedClassesAndMethodMap.get(testNameId); - localHolder.doUpdateFromDiff(testNameId, result, previousClassData, moduleName != null ? localHolder.myModuleEnumerator.enumerate(new TestInfoHolder.ModuleId(moduleName, frameworkId)) : null); + byte frameworkId) { + executeUnderLock(holder -> { + holder.updateTestData(testClassName, testMethodName, usedMethods, moduleName, frameworkId); return null; }); } - private TestInfoHolder getHolder() { - TestInfoHolder holder = myHolder; + private DiscoveredTestDataHolder getHolder() { + DiscoveredTestDataHolder holder = myHolder; if (holder == null) { synchronized (myLock) { holder = myHolder; - if (holder == null && myBasePath != null) myHolder = holder = new TestInfoHolder(myBasePath, false, myLock); + if (holder == null && myBasePath != null) { + myHolder = holder = new DiscoveredTestDataHolder(myBasePath); + } } } return holder; } - private void thingsWentWrongLetsReinitialize(@Nullable TestInfoHolder holder, Throwable throwable) throws IOException { - LOG.error("Unexpected problem", throwable); - if (holder != null) holder.dispose(); - PathKt.delete(TestInfoHolder.getVersionFile(myBasePath)); - - myHolder = null; - if (throwable instanceof IOException) throw (IOException)throwable; - } - - private R withTestDataHolder(ThrowableConvertor action) throws IOException { + private R executeUnderLock(ThrowableConvertor action) { synchronized (myLock) { - TestInfoHolder holder = getHolder(); + DiscoveredTestDataHolder holder = getHolder(); if (holder == null || holder.isDisposed()) return null; try { return action.convert(holder); } catch (Throwable throwable) { - thingsWentWrongLetsReinitialize(holder, throwable); + LOG.error("Unexpected problem", throwable); + holder.dispose(); + PathKt.delete(myBasePath); + myHolder = null; } return null; } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoverySearchHelper.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoverySearchHelper.java index 5577bf4a1f1f..3b67fe39e6cc 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoverySearchHelper.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoverySearchHelper.java @@ -62,14 +62,11 @@ public class TestDiscoverySearchHelper { if (containingClass != null && containingClass.getParent() == psiFile) { final String classQualifiedName = containingClass.getQualifiedName(); final String changedMethodName = changedMethod.getName(); - try { - if (classQualifiedName != null && - (position == null && TestFrameworks.detectFramework(containingClass) != null || - position != null && !discoveryIndex.hasTestTrace(classQualifiedName, changedMethodName, frameworkId))) { - patterns.add(classQualifiedName + "," + changedMethodName); - } + if (classQualifiedName != null && + (position == null && TestFrameworks.detectFramework(containingClass) != null || + position != null && !discoveryIndex.hasTestTrace(classQualifiedName, changedMethodName, frameworkId))) { + patterns.add(classQualifiedName + "," + changedMethodName); } - catch (IOException ignore) {} } } } diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestInfoHolder.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestInfoHolder.java deleted file mode 100644 index ae5d0d7a3742..000000000000 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestInfoHolder.java +++ /dev/null @@ -1,557 +0,0 @@ -// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -package com.intellij.execution.testDiscovery; - -import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon; -import com.intellij.util.io.*; -import com.intellij.util.io.DataOutputStream; -import gnu.trove.TIntArrayList; -import gnu.trove.TIntHashSet; -import gnu.trove.TIntObjectHashMap; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.io.*; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ScheduledFuture; - -final class TestInfoHolder { - final PersistentHashMap myMethodQNameToTestNames; - final PersistentHashMap> myTestNameToUsedClassesAndMethodMap; - final PersistentHashMap myTestNameToNearestModule; - final PersistentStringEnumerator myClassEnumerator; - final PersistentStringEnumerator myMethodEnumerator; - final PersistentEnumeratorDelegate myTestEnumerator; - final PersistentEnumeratorDelegate myModuleEnumerator; - final List myConstructedDataFiles = new ArrayList<>(6); - - private ScheduledFuture myFlushingFuture; - private boolean myDisposed; - private final Object myLock; - - private static final int VERSION = 7; - - TestInfoHolder(@NotNull Path basePath, boolean readOnly, Object lock) { - myLock = lock; - final Path versionFile = getVersionFile(basePath); - PathKt.createDirectories(basePath); - final File methodQNameToTestNameFile = basePath.resolve("methodQNameToTestName.data").toFile(); - final File testNameToUsedClassesAndMethodMapFile = basePath.resolve("testToCalledMethodNames.data").toFile(); - final File classNameEnumeratorFile = basePath.resolve("classNameEnumerator.data").toFile(); - final File methodNameEnumeratorFile = basePath.resolve("methodNameEnumerator.data").toFile(); - final File testNameEnumeratorFile = basePath.resolve("testNameEnumerator.data").toFile(); - final File moduleNameEnumeratorFile = basePath.resolve("moduleNameEnumerator.data").toFile(); - final File testNameToNearestModuleFile = basePath.resolve("testNameToNearestModule.data").toFile(); - - try { - int version = readVersion(versionFile); - if (version != VERSION) { - TestDiscoveryIndex.LOG.info(version != -1 - ? "TestDiscoveryIndex was rewritten due to version change" - : "TestDiscoveryIndex is not exist. Empty index is created"); - assert !readOnly; - deleteAllIndexDataFiles(methodQNameToTestNameFile, - testNameToUsedClassesAndMethodMapFile, - classNameEnumeratorFile, - methodNameEnumeratorFile, - testNameEnumeratorFile, moduleNameEnumeratorFile, - testNameToNearestModuleFile); - - writeVersion(versionFile); - } - - PersistentHashMap methodQNameToTestNames; - PersistentHashMap> testNameToUsedClassesAndMethodMap; - PersistentHashMap testNameToNearestModule; - PersistentStringEnumerator classNameEnumerator; - PersistentStringEnumerator methodEnumerator; - PersistentEnumeratorDelegate testEnumerator; - PersistentEnumeratorDelegate moduleNameEnumerator; - - int iterations = 0; - - while (true) { - ++iterations; - - try { - methodQNameToTestNames = new PersistentHashMap( - methodQNameToTestNameFile, - MethodQNameSerializer.INSTANCE, - new TestNamesExternalizer() - ) { - @Override - protected boolean isReadOnly() { - return readOnly; - } - }; - myConstructedDataFiles.add(methodQNameToTestNames); - - testNameToUsedClassesAndMethodMap = new PersistentHashMap>( - testNameToUsedClassesAndMethodMapFile, - EnumeratorIntegerDescriptor.INSTANCE, - new ClassesAndMethodsMapDataExternalizer() - ) { - @Override - protected boolean isReadOnly() { - return readOnly; - } - }; - myConstructedDataFiles.add(testNameToUsedClassesAndMethodMap); - - testNameToNearestModule = new PersistentHashMap<>(testNameToNearestModuleFile, - MethodQNameSerializer.INSTANCE, - new TestNamesExternalizer()); - myConstructedDataFiles.add(testNameToNearestModule); - - classNameEnumerator = new PersistentStringEnumerator(classNameEnumeratorFile, true); - myConstructedDataFiles.add(classNameEnumerator); - - methodEnumerator = new PersistentStringEnumerator(methodNameEnumeratorFile, true); - myConstructedDataFiles.add(methodEnumerator); - - moduleNameEnumerator = new PersistentEnumeratorDelegate<>(moduleNameEnumeratorFile, ModuleId.DESCRIPTOR, 64); - myConstructedDataFiles.add(moduleNameEnumerator); - - testEnumerator = new PersistentEnumeratorDelegate<>(testNameEnumeratorFile, TestId.DESCRIPTOR, 1024 * 4); - myConstructedDataFiles.add(testEnumerator); - - break; - } - catch (Throwable throwable) { - TestDiscoveryIndex.LOG.info("TestDiscoveryIndex problem", throwable); - closeAllConstructedFiles(true); - myConstructedDataFiles.clear(); - - deleteAllIndexDataFiles(methodQNameToTestNameFile, testNameToUsedClassesAndMethodMapFile, classNameEnumeratorFile, - methodNameEnumeratorFile, - testNameEnumeratorFile, moduleNameEnumeratorFile, testNameToNearestModuleFile); - // try another time - } - - if (iterations >= 3) { - TestDiscoveryIndex.LOG.error("Unexpected circular initialization problem"); - assert false; - } - } - - myMethodQNameToTestNames = methodQNameToTestNames; - myTestNameToUsedClassesAndMethodMap = testNameToUsedClassesAndMethodMap; - myTestNameToNearestModule = testNameToNearestModule; - myClassEnumerator = classNameEnumerator; - myMethodEnumerator = methodEnumerator; - myTestEnumerator = testEnumerator; - myModuleEnumerator = moduleNameEnumerator; - - myFlushingFuture = FlushingDaemon.everyFiveSeconds(() -> { - synchronized (myLock) { - if (myDisposed) { - myFlushingFuture.cancel(false); - return; - } - for (PersistentEnumeratorDelegate dataFile : myConstructedDataFiles) { - if (dataFile.isDirty()) { - dataFile.force(); - } - } - } - }); - } - catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - private void closeAllConstructedFiles(boolean ignoreCloseProblem) { - for (Closeable closeable : myConstructedDataFiles) { - try { - closeable.close(); - } - catch (Throwable throwable) { - if (!ignoreCloseProblem) throw new RuntimeException(throwable); - } - } - } - - private static void deleteAllIndexDataFiles(File... files) { - for (File file : files) { - IOUtil.deleteAllFilesStartingWith(file); - } - } - - private static void writeVersion(@NotNull Path versionFile) throws IOException { - try (final DataOutputStream versionOut = new DataOutputStream(PathKt.outputStream(versionFile))) { - DataInputOutputUtil.writeINT(versionOut, VERSION); - } - } - - private static int readVersion(@NotNull Path versionFile) throws IOException { - InputStream inputStream = PathKt.inputStreamIfExists(versionFile); - if (inputStream == null) { - return -1; - } - try (DataInputStream versionInput = new DataInputStream(inputStream)) { - return DataInputOutputUtil.readINT(versionInput); - } - } - - void dispose() { - assert Thread.holdsLock(myLock); - try { - closeAllConstructedFiles(false); - } - finally { - myDisposed = true; - } - } - - private static final int REMOVED_MARKER = -1; - - void doUpdateFromDiff(final int testNameId, - @Nullable TIntObjectHashMap classData, - @Nullable TIntObjectHashMap previousClassData, - @Nullable Integer moduleId) throws IOException { - ValueDiff valueDiff = new ValueDiff(classData, previousClassData); - - if (valueDiff.hasRemovedDelta()) { - for (int classQName : valueDiff.myRemovedClassData.keys()) { - for (int methodName : valueDiff.myRemovedClassData.get(classQName).toNativeArray()) { - myMethodQNameToTestNames.appendData(createKey(classQName, methodName), - dataOutput -> { - DataInputOutputUtil.writeINT(dataOutput, REMOVED_MARKER); - DataInputOutputUtil.writeINT(dataOutput, testNameId); - } - ); - } - } - } - - if (valueDiff.hasAddedDelta()) { - for (int classQName : valueDiff.myAddedOrChangedClassData.keys()) { - for (int methodName : valueDiff.myAddedOrChangedClassData.get(classQName).toNativeArray()) { - myMethodQNameToTestNames.appendData(createKey(classQName, methodName), - dataOutput -> DataInputOutputUtil.writeINT(dataOutput, testNameId)); - if (moduleId != null) { - myTestNameToNearestModule.appendData(createKey(classQName, methodName), - dataOutput -> DataInputOutputUtil.writeINT(dataOutput, moduleId)); - } - } - } - } - - if ((valueDiff.hasAddedDelta() || valueDiff.hasRemovedDelta())) { - if (classData != null) { - myTestNameToUsedClassesAndMethodMap.put(testNameId, classData); - } - else { - myTestNameToUsedClassesAndMethodMap.remove(testNameId); - } - } - } - - public boolean isDisposed() { - return myDisposed; - } - - public static boolean isValidPath(@NotNull Path path) { - try { - return readVersion(getVersionFile(path)) == VERSION; - } - catch (IOException ex) { - return false; - } - } - - private static class TestNamesExternalizer implements DataExternalizer { - public void save(@NotNull DataOutput dataOutput, TIntArrayList testNameIds) throws IOException { - for (int testNameId : testNameIds.toNativeArray()) DataInputOutputUtil.writeINT(dataOutput, testNameId); - } - - public TIntArrayList read(@NotNull DataInput dataInput) throws IOException { - TIntHashSet result = new TIntHashSet(); - - while (((InputStream)dataInput).available() > 0) { - int id = DataInputOutputUtil.readINT(dataInput); - if (REMOVED_MARKER == id) { - id = DataInputOutputUtil.readINT(dataInput); - if(!result.remove(id)) { - result.add(-id); - } - } - else { - result.add(id); - } - } - - return new TIntArrayList(result.toArray()); - } - } - - private static class ClassesAndMethodsMapDataExternalizer implements DataExternalizer> { - public void save(@NotNull final DataOutput dataOutput, TIntObjectHashMap classAndMethodsMap) - throws IOException { - DataInputOutputUtil.writeINT(dataOutput, classAndMethodsMap.size()); - final int[] classNameIds = classAndMethodsMap.keys(); - Arrays.sort(classNameIds); - - int prevClassNameId = 0; - for (int classNameId : classNameIds) { - DataInputOutputUtil.writeINT(dataOutput, classNameId - prevClassNameId); - TIntArrayList value = classAndMethodsMap.get(classNameId); - DataInputOutputUtil.writeINT(dataOutput, value.size()); - - final int[] methodNameIds = value.toNativeArray(); - Arrays.sort(methodNameIds); - int prevMethodNameId = 0; - for (int methodNameId : methodNameIds) { - DataInputOutputUtil.writeINT(dataOutput, methodNameId - prevMethodNameId); - prevMethodNameId = methodNameId; - } - prevClassNameId = classNameId; - } - } - - public TIntObjectHashMap read(@NotNull DataInput dataInput) throws IOException { - int numberOfClasses = DataInputOutputUtil.readINT(dataInput); - TIntObjectHashMap result = new TIntObjectHashMap<>(); - int prevClassNameId = 0; - - while (numberOfClasses-- > 0) { - int classNameId = DataInputOutputUtil.readINT(dataInput) + prevClassNameId; - int numberOfMethods = DataInputOutputUtil.readINT(dataInput); - TIntArrayList methodNameIds = new TIntArrayList(numberOfMethods); - - int prevMethodNameId = 0; - while (numberOfMethods-- > 0) { - final int methodNameId = DataInputOutputUtil.readINT(dataInput) + prevMethodNameId; - methodNameIds.add(methodNameId); - prevMethodNameId = methodNameId; - } - - result.put(classNameId, methodNameIds); - prevClassNameId = classNameId; - } - return result; - } - } - - private static class MethodQNameSerializer implements KeyDescriptor { - public static final MethodQNameSerializer INSTANCE = new MethodQNameSerializer(); - - @Override - public void save(@NotNull DataOutput out, Long value) throws IOException { - out.writeLong(value); - } - - @Override - public Long read(@NotNull DataInput in) throws IOException { - return in.readLong(); - } - - @Override - public int getHashCode(Long value) { - return value.hashCode(); - } - - @Override - public boolean isEqual(Long val1, Long val2) { - return val1.equals(val2); - } - } - - @NotNull - static Path getVersionFile(Path path) { - return path.resolve("index.version"); - } - - static long createKey(int classQName, int methodName) { - return ((long)classQName << 32) | methodName; - } - - static class ValueDiff { - final TIntObjectHashMap myAddedOrChangedClassData; - final TIntObjectHashMap myRemovedClassData; - - ValueDiff(@Nullable TIntObjectHashMap classData, @Nullable TIntObjectHashMap previousClassData) { - TIntObjectHashMap addedOrChangedClassData = classData; - TIntObjectHashMap removedClassData = previousClassData; - - if (previousClassData != null && !previousClassData.isEmpty()) { - removedClassData = new TIntObjectHashMap<>(); - addedOrChangedClassData = new TIntObjectHashMap<>(); - - if (classData != null) { - for (int classQName : classData.keys()) { - TIntArrayList currentMethods = classData.get(classQName); - TIntArrayList previousMethods = previousClassData.get(classQName); - - if (previousMethods == null) { - addedOrChangedClassData.put(classQName, currentMethods); - continue; - } - - final int[] previousMethodIds = previousMethods.toNativeArray(); - TIntHashSet previousMethodsSet = new TIntHashSet(previousMethodIds); - final int[] currentMethodIds = currentMethods.toNativeArray(); - TIntHashSet currentMethodsSet = new TIntHashSet(currentMethodIds); - currentMethodsSet.removeAll(previousMethodIds); - previousMethodsSet.removeAll(currentMethodIds); - - if (!currentMethodsSet.isEmpty()) { - addedOrChangedClassData.put(classQName, new TIntArrayList(currentMethodsSet.toArray())); - } - if (!previousMethodsSet.isEmpty()) { - removedClassData.put(classQName, new TIntArrayList(previousMethodsSet.toArray())); - } - } - } - if (classData != null) { - for (int classQName : previousClassData.keys()) { - if (classData.containsKey(classQName)) continue; - - TIntArrayList previousMethods = previousClassData.get(classQName); - removedClassData.put(classQName, previousMethods); - } - } - } - - myAddedOrChangedClassData = addedOrChangedClassData; - myRemovedClassData = removedClassData; - } - - public boolean hasRemovedDelta() { - return myRemovedClassData != null && !myRemovedClassData.isEmpty(); - } - - public boolean hasAddedDelta() { - return myAddedOrChangedClassData != null && !myAddedOrChangedClassData.isEmpty(); - } - } - - @NotNull - TestId createTestId(String className, String methodName, byte frameworkPrefix) throws IOException { - return new TestId(myClassEnumerator.enumerate(className), myMethodEnumerator.enumerate(methodName), frameworkPrefix); - } - - static class ModuleId { - @NotNull - private final String myModuleName; - private final byte myFrameworkId; - - ModuleId(@NotNull String name, byte id) { - myModuleName = name; - myFrameworkId = id; - } - - @NotNull - String getModuleName() { - return myModuleName; - } - - byte getFrameworkId() { - return myFrameworkId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ModuleId id = (ModuleId)o; - return myFrameworkId == id.myFrameworkId && - Objects.equals(myModuleName, id.myModuleName); - } - - @Override - public int hashCode() { - return Objects.hash(myModuleName, myFrameworkId); - } - - private static final KeyDescriptor DESCRIPTOR = new KeyDescriptor() { - @Override - public int getHashCode(ModuleId id) { - return id.hashCode(); - } - - @Override - public boolean isEqual(ModuleId id1, ModuleId id2) { - return id1.equals(id2); - } - - @Override - public void save(@NotNull DataOutput out, ModuleId id) throws IOException { - IOUtil.writeUTF(out, id.getModuleName()); - out.writeByte(id.getFrameworkId()); - } - - @Override - public ModuleId read(@NotNull DataInput in) throws IOException { - return new ModuleId(IOUtil.readUTF(in), in.readByte()); - } - }; - } - - static class TestId { - private final int myClassId; - private final int myMethodId; - private final byte myFrameworkId; - - TestId(int classId, int methodId, byte id) { - myClassId = classId; - myMethodId = methodId; - myFrameworkId = id; - } - - int getClassId() { - return myClassId; - } - - int getMethodId() { - return myMethodId; - } - - byte getFrameworkId() { - return myFrameworkId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TestId id = (TestId)o; - return myClassId == id.myClassId && - myMethodId == id.myMethodId && - myFrameworkId == id.myFrameworkId; - } - - @Override - public int hashCode() { - return Objects.hash(myClassId, myMethodId, myFrameworkId); - } - - private static final KeyDescriptor DESCRIPTOR = new KeyDescriptor() { - @Override - public int getHashCode(TestId id) { - return id.hashCode(); - } - - @Override - public boolean isEqual(TestId id1, TestId id2) { - return id1.equals(id2); - } - - @Override - public void save(@NotNull DataOutput out, TestId id) throws IOException { - DataInputOutputUtil.writeINT(out, id.getClassId()); - DataInputOutputUtil.writeINT(out, id.getMethodId()); - out.writeByte(id.getFrameworkId()); - } - - @Override - public TestId read(@NotNull DataInput in) throws IOException { - return new TestId(DataInputOutputUtil.readINT(in), DataInputOutputUtil.readINT(in), in.readByte()); - } - }; - } -} diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java new file mode 100644 index 000000000000..1752368dfb66 --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestDataHolder.java @@ -0,0 +1,237 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import com.intellij.openapi.Disposable; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Disposer; +import com.intellij.openapi.util.LowMemoryWatcher; +import com.intellij.util.containers.MultiMap; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.indexing.ValueContainer; +import com.intellij.util.io.*; +import com.intellij.util.io.DataOutputStream; +import gnu.trove.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.*; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +public final class DiscoveredTestDataHolder { + private static final Logger LOG = Logger.getInstance(DiscoveredTestDataHolder.class); + + private final DiscoveredTestsIndex myDiscoveredTestsIndex; + private final TestModuleIndex myTestModuleIndex; + + private final PersistentStringEnumerator myClassEnumerator; + private final PersistentStringEnumerator myMethodEnumerator; + private final PersistentEnumeratorDelegate myTestEnumerator; + private final PersistentObjectSeq myConstructedDataFiles = new PersistentObjectSeq(); + + private boolean myDisposed; + private final Disposable myDisposable = Disposer.newDisposable(); + + static final int VERSION = 8; + + public DiscoveredTestDataHolder(@NotNull Path basePath) { + final Path versionFile = getVersionFile(basePath); + PathKt.createDirectories(basePath); + final File discoveredTestsIndexFile = basePath.resolve("discoveredTests.index").toFile(); + + final File classNameEnumeratorFile = basePath.resolve("className.enum").toFile(); + final File methodNameEnumeratorFile = basePath.resolve("methodName.enum").toFile(); + final File testNameEnumeratorFile = basePath.resolve("testName.enum").toFile(); + + try { + int version = readVersion(versionFile); + if (version != VERSION) { + LOG.info(version != -1 + ? "TestDiscoveryIndex was rewritten due to version change" + : "TestDiscoveryIndex is not exist. Empty index is created"); + PathKt.delete(basePath); + writeVersion(versionFile); + } + + DiscoveredTestsIndex discoveredTestsIndex; + TestModuleIndex testModuleIndex; + PersistentStringEnumerator classNameEnumerator; + PersistentStringEnumerator methodEnumerator; + PersistentEnumeratorDelegate testEnumerator; + + int iterations = 0; + + while (true) { + ++iterations; + + try { + discoveredTestsIndex = new DiscoveredTestsIndex(discoveredTestsIndexFile); + myConstructedDataFiles.add(discoveredTestsIndex); + + testModuleIndex = new TestModuleIndex(basePath, myConstructedDataFiles); + + classNameEnumerator = new PersistentStringEnumerator(classNameEnumeratorFile, true); + myConstructedDataFiles.add(classNameEnumerator); + + methodEnumerator = new PersistentStringEnumerator(methodNameEnumeratorFile, true); + myConstructedDataFiles.add(methodEnumerator); + + testEnumerator = new PersistentEnumeratorDelegate<>(testNameEnumeratorFile, TestId.DESCRIPTOR, 1024 * 4); + myConstructedDataFiles.add(testEnumerator); + + break; + } + catch (Throwable throwable) { + LOG.info("TestDiscoveryIndex problem", throwable); + myConstructedDataFiles.close(true); + myConstructedDataFiles.clear(); + + PathKt.delete(basePath); + // try another time + } + + if (iterations >= 3) { + LOG.error("Unexpected circular initialization problem"); + assert false; + } + } + + myDiscoveredTestsIndex = discoveredTestsIndex; + myTestModuleIndex = testModuleIndex; + myClassEnumerator = classNameEnumerator; + myMethodEnumerator = methodEnumerator; + myTestEnumerator = testEnumerator; + + LowMemoryWatcher.register(() -> myConstructedDataFiles.flush(), myDisposable); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + private static void writeVersion(@NotNull Path versionFile) throws IOException { + try (final DataOutputStream versionOut = new DataOutputStream(PathKt.outputStream(versionFile))) { + DataInputOutputUtil.writeINT(versionOut, VERSION); + } + } + + private static int readVersion(@NotNull Path versionFile) throws IOException { + InputStream inputStream = PathKt.inputStreamIfExists(versionFile); + if (inputStream == null) { + return -1; + } + try (DataInputStream versionInput = new DataInputStream(inputStream)) { + return DataInputOutputUtil.readINT(versionInput); + } + } + + public void flush() { + myConstructedDataFiles.flush(); + } + + public void dispose() { + try { + Disposer.dispose(myDisposable); + myConstructedDataFiles.close(false); + } + finally { + myDisposed = true; + } + } + + public boolean hasTestTrace(@NotNull String testClassName, @NotNull String testMethodName, byte frameworkId) throws IOException { + int testId = myTestEnumerator.tryEnumerate(createTestId(testClassName, testMethodName, frameworkId)); + return testId != 0 && myDiscoveredTestsIndex.containsDataFrom(testId); + } + + public void removeTestTrace(@NotNull String testClassName, @NotNull String testMethodName, byte frameworkId) throws IOException { + int testId = myTestEnumerator.tryEnumerate(createTestId(testClassName, testMethodName, frameworkId)); + if (testId != 0) { + myDiscoveredTestsIndex.update(testId, null).compute(); + myTestModuleIndex.removeTest(testId); + } + } + + @NotNull + public Collection getTestModulesByMethodName(@NotNull String testClassName, @NotNull String testMethodName, byte frameworkId) throws IOException { + int testId = myTestEnumerator.tryEnumerate(createTestId(testClassName, testMethodName, frameworkId)); + if (testId != 0) { + return myTestModuleIndex.getTestRunModules(testId); + } + return Collections.emptySet(); + } + + public void updateTestData(@NotNull String testClassName, + @NotNull String testMethodName, + @NotNull MultiMap usedMethods, + @Nullable String moduleName, + byte frameworkId) throws IOException { + + final int testNameId = myTestEnumerator.enumerate(createTestId(testClassName, testMethodName, frameworkId)); + TLongHashSet result = new TLongHashSet(); + for (Map.Entry> e : usedMethods.entrySet()) { + int classId = myClassEnumerator.enumerate(e.getKey()); + for (String methodName : e.getValue()) { + long key = createKey(classId, myMethodEnumerator.enumerate(methodName)); + result.add(key); + } + } + myDiscoveredTestsIndex.update(testNameId, new DiscoveredTestsIndex.UsedMethods(result)).compute(); + myTestModuleIndex.appendModuleData(testNameId, moduleName); + } + + @NotNull + public MultiMap getTestsByMethodName(@NotNull String classFQName, @NotNull String methodName, byte frameworkId) throws IOException { + int methodId = myMethodEnumerator.tryEnumerate(methodName); + if (methodId == 0) return MultiMap.empty(); + int classId = myClassEnumerator.tryEnumerate(classFQName); + if (classId == 0) return MultiMap.empty(); + try { + MultiMap result = new MultiMap<>(); + IOException[] exception = {null}; + myDiscoveredTestsIndex.getData(createKey(classId, methodId)).forEach(new ValueContainer.ContainerAction() { + @Override + public boolean perform(int testId, Void value) { + try { + TestId test = myTestEnumerator.valueOf(testId); + if (test.getFrameworkId() == frameworkId) { + String testClassName = myClassEnumerator.valueOf(test.getClassId()); + String testMethodName = myMethodEnumerator.valueOf(test.getMethodId()); + result.putValue(testClassName, testMethodName); + } + } + catch (IOException e) { + exception[0] = e; + return false; + } + return true; + } + }); + if (exception[0] != null) throw exception[0]; + return result; + } + catch (StorageException e) { + throw new IOException(e); + } + } + + public boolean isDisposed() { + return myDisposed; + } + + @NotNull + static Path getVersionFile(Path path) { + return path.resolve("index.version"); + } + + public static long createKey(int classQName, int methodName) { + return ((long)classQName << 32) | methodName; + } + + @NotNull + public TestId createTestId(String className, String methodName, byte frameworkPrefix) throws IOException { + return new TestId(myClassEnumerator.enumerate(className), myMethodEnumerator.enumerate(methodName), frameworkPrefix); + } +} diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java new file mode 100644 index 000000000000..97786f7e1a37 --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/DiscoveredTestsIndex.java @@ -0,0 +1,228 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.util.indexing.DataIndexer; +import com.intellij.util.indexing.IndexExtension; +import com.intellij.util.indexing.IndexId; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.indexing.impl.*; +import com.intellij.util.io.*; +import gnu.trove.TLongHashSet; +import gnu.trove.TLongIterator; +import gnu.trove.TLongProcedure; +import org.jetbrains.annotations.NotNull; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; +import java.util.*; + +public class DiscoveredTestsIndex extends MapReduceIndex { + protected DiscoveredTestsIndex(@NotNull File file) throws IOException { + super(INDEX_EXTENSION, new MyIndexStorage(file), new MyForwardIndex() { + @NotNull + @Override + public PersistentHashMap createMap() throws IOException { + return new PersistentHashMap<>(new File(file, "forward.idx"), EnumeratorIntegerDescriptor.INSTANCE, UsedMethods.USED_METHODS_DATA_EXTERNALIZER); + } + }); + } + + @Override + public void checkCanceled() { + ProgressManager.checkCanceled(); + } + + @Override + protected void requestRebuild(Throwable e) { + //TODO index corrupted + } + + public boolean containsDataFrom(int testId) throws IOException { + return ((MyForwardIndex)myForwardIndex).containsDataFrom(testId); + } + + private static class MyIndexStorage extends MapIndexStorage { + protected MyIndexStorage(@NotNull File storageFile) throws IOException { + super(storageFile, MethodQNameSerializer.INSTANCE, VoidDataExternalizer.INSTANCE, 4 * 1024, false); + } + + @Override + protected void checkCanceled() { + ProgressManager.checkCanceled(); + } + } + + private static final IndexExtension INDEX_EXTENSION = new IndexExtension() { + @NotNull + @Override + public IndexId getName() { + return IndexId.create("jvm.discovered.tests"); + } + + @NotNull + @Override + public DataIndexer getIndexer() {return inputData -> inputData;} + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return MethodQNameSerializer.INSTANCE; + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return VoidDataExternalizer.INSTANCE; + } + + @Override + public int getVersion() { + return DiscoveredTestDataHolder.VERSION; + } + }; + + + private abstract static class MyForwardIndex extends MapBasedForwardIndex { + protected MyForwardIndex() throws IOException { + super(INDEX_EXTENSION); + } + + @Override + protected InputDataDiffBuilder getDiffBuilder(int inputId, UsedMethods oldData) { + return new InputDataDiffBuilder(inputId) { + @Override + public boolean differentiate(@NotNull Map newData, + @NotNull KeyValueUpdateProcessor addProcessor, + @NotNull KeyValueUpdateProcessor updateProcessor, + @NotNull RemovedKeyProcessor removeProcessor) throws StorageException { + boolean[] updated = {false}; + StorageException[] exception = {null}; + if (oldData != null) { + oldData.forEach(m -> { + if (!newData.containsKey(m)) { + try { + removeProcessor.process(m, myInputId); + if (!updated[0]) { + updated[0] = true; + } + } + catch (StorageException e) { + exception[0] = e; + return false; + } + } + return true; + }); + } + if (exception[0] != null) { + throw exception[0]; + } + + ((UsedMethods)newData).forEach(m -> { + if (oldData == null || !oldData.containsKey(m)) { + try { + addProcessor.process(m, null, myInputId); + if (!updated[0]) { + updated[0] = true; + } + } + catch (StorageException e) { + exception[0] = e; + return false; + } + } + return true; + }); + if (exception[0] != null) { + throw exception[0]; + } + return updated[0]; + } + }; + } + + @Override + protected UsedMethods convertToMapValueType(int inputId, Map map) { + return (UsedMethods) map; + } + + public boolean containsDataFrom(int testId) throws IOException { + return getInput(testId) != null; + } + } + + static class UsedMethods extends AbstractMap { + private final TLongHashSet myTestUsedMethods; + + UsedMethods(TLongHashSet methods) {myTestUsedMethods = methods;} + + public boolean contains(long method) { + return myTestUsedMethods.contains(method); + } + + public boolean forEach(@NotNull TLongProcedure methodProcedure) { + return myTestUsedMethods.forEach(methodProcedure); + } + + @Override + public Set> entrySet() { + return new AbstractSet>() { + @Override + public Iterator> iterator() { + TLongIterator methodIterator = myTestUsedMethods.iterator(); + return new Iterator>() { + @Override + public boolean hasNext() { + return methodIterator.hasNext(); + } + + @Override + public Entry next() { + return new SimpleEntry<>(methodIterator.next(), null); + } + }; + + } + + @Override + public int size() { + return myTestUsedMethods.size(); + } + }; + } + + private static final DataExternalizer USED_METHODS_DATA_EXTERNALIZER = new DataExternalizer() { + @Override + public void save(@NotNull DataOutput out, UsedMethods value) throws IOException { + DataInputOutputUtil.writeINT(out, value.size()); + IOException[] exception = {null}; + value.forEach(m -> { + try { + MethodQNameSerializer.INSTANCE.save(out, m); + } + catch (IOException e) { + exception[0] = null; + return false; + } + return true; + }); + if (exception[0] != null) { + throw exception[0]; + } + } + + @Override + public UsedMethods read(@NotNull DataInput in) throws IOException { + int size = DataInputOutputUtil.readINT(in); + TLongHashSet result = new TLongHashSet(); + for (int i = 0; i < size; i++) { + result.add(MethodQNameSerializer.INSTANCE.read(in)); + } + return new UsedMethods(result); + } + }; + } +} diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/MethodQNameSerializer.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/MethodQNameSerializer.java new file mode 100644 index 000000000000..a58a9a1cfc9b --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/MethodQNameSerializer.java @@ -0,0 +1,35 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import com.intellij.util.io.KeyDescriptor; +import org.jetbrains.annotations.NotNull; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +class MethodQNameSerializer implements KeyDescriptor { + public static final MethodQNameSerializer INSTANCE = new MethodQNameSerializer(); + + private MethodQNameSerializer() {} + + @Override + public void save(@NotNull DataOutput out, Long value) throws IOException { + out.writeLong(value); + } + + @Override + public Long read(@NotNull DataInput in) throws IOException { + return in.readLong(); + } + + @Override + public int getHashCode(Long value) { + return value.hashCode(); + } + + @Override + public boolean isEqual(Long val1, Long val2) { + return val1.equals(val2); + } +} diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/PersistentObjectSeq.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/PersistentObjectSeq.java new file mode 100644 index 000000000000..0a40a4bed03b --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/PersistentObjectSeq.java @@ -0,0 +1,81 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.InvertedIndex; +import com.intellij.util.io.PersistentEnumeratorDelegate; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +class PersistentObjectSeq { + private static final Logger LOG = Logger.getInstance(PersistentObjectSeq.class); + + interface PersistentObject { + void flush() throws Exception; + + void close() throws IOException; + } + + private final Collection myObjects = ContainerUtil.newConcurrentSet(); + + public void add(InvertedIndex index) { + myObjects.add(new PersistentObject() { + @Override + public void flush() throws Exception { + index.flush(); + } + + @Override + public void close() { + index.dispose(); + } + }); + } + + public void add(PersistentEnumeratorDelegate enumerator) { + myObjects.add(new PersistentObject() { + @Override + public void flush() { + if (enumerator.isDirty()) { + enumerator.force(); + } + } + + @Override + public void close() throws IOException { + enumerator.close(); + } + }); + } + + public void close(boolean ignoreCloseProblem) { + for (PersistentObject object : myObjects) { + try { + object.close(); + } + catch (Throwable throwable) { + if (!ignoreCloseProblem) throw new RuntimeException(throwable); + } + } + } + + public void flush() { + for (PersistentObject object : myObjects) { + try { + object.flush(); + } + catch (Exception e) { + LOG.error(e); + } + } + } + + public void clear() { + myObjects.clear(); + } +} diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestId.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestId.java new file mode 100644 index 000000000000..783273b88524 --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestId.java @@ -0,0 +1,74 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import com.intellij.util.io.DataInputOutputUtil; +import com.intellij.util.io.KeyDescriptor; +import org.jetbrains.annotations.NotNull; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Objects; + +public class TestId { + private final int myClassId; + private final int myMethodId; + private final byte myFrameworkId; + + TestId(int classId, int methodId, byte id) { + myClassId = classId; + myMethodId = methodId; + myFrameworkId = id; + } + + public int getClassId() { + return myClassId; + } + + public int getMethodId() { + return myMethodId; + } + + public byte getFrameworkId() { + return myFrameworkId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestId id = (TestId)o; + return myClassId == id.myClassId && + myMethodId == id.myMethodId && + myFrameworkId == id.myFrameworkId; + } + + @Override + public int hashCode() { + return Objects.hash(myClassId, myMethodId, myFrameworkId); + } + + static final KeyDescriptor DESCRIPTOR = new KeyDescriptor() { + @Override + public int getHashCode(TestId id) { + return id.hashCode(); + } + + @Override + public boolean isEqual(TestId id1, TestId id2) { + return id1.equals(id2); + } + + @Override + public void save(@NotNull DataOutput out, TestId id) throws IOException { + DataInputOutputUtil.writeINT(out, id.getClassId()); + DataInputOutputUtil.writeINT(out, id.getMethodId()); + out.writeByte(id.getFrameworkId()); + } + + @Override + public TestId read(@NotNull DataInput in) throws IOException { + return new TestId(DataInputOutputUtil.readINT(in), DataInputOutputUtil.readINT(in), in.readByte()); + } + }; +} diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestModuleIndex.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestModuleIndex.java new file mode 100644 index 000000000000..87dca85787fa --- /dev/null +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/indices/TestModuleIndex.java @@ -0,0 +1,80 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.execution.testDiscovery.indices; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.util.io.*; +import gnu.trove.TIntArrayList; +import gnu.trove.TIntHashSet; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.*; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class TestModuleIndex { + private static final Logger LOG = Logger.getInstance(TestModuleIndex.class); + + private final PersistentHashMap myTestNameToRunModule; + private final PersistentEnumeratorDelegate myModuleNameEnumerator; + + public TestModuleIndex(@NotNull Path basePath, @NotNull PersistentObjectSeq persistentObjectSeq) throws IOException { + File moduleNameEnumeratorFile = basePath.resolve("moduleName.enum").toFile(); + myModuleNameEnumerator = new PersistentEnumeratorDelegate<>(moduleNameEnumeratorFile, EnumeratorStringDescriptor.INSTANCE, 64); + persistentObjectSeq.add(myModuleNameEnumerator); + + File testModuleIndexFile = basePath.resolve("testModule.index").toFile(); + myTestNameToRunModule = new PersistentHashMap<>(testModuleIndexFile, EnumeratorIntegerDescriptor.INSTANCE, + new IntSeqExternalizer()); + } + + void appendModuleData(int testId, @Nullable String moduleName) throws IOException { + if (moduleName != null) { + int moduleId = myModuleNameEnumerator.enumerate(moduleName); + TIntArrayList previousRunModules = myTestNameToRunModule.get(moduleId); + if (previousRunModules != null && previousRunModules.contains(moduleId)) { + return; + } + myTestNameToRunModule.appendData(testId, out -> DataInputOutputUtil.writeINT(out, moduleId)); + } + } + + @NotNull + Collection getTestRunModules(int testId) throws IOException { + TIntArrayList moduleIds = myTestNameToRunModule.get(testId); + if (moduleIds == null) return Collections.emptySet(); + List result = new ArrayList<>(moduleIds.size()); + for (int i = 0; i < moduleIds.size(); i++) { + int moduleId = moduleIds.get(i); + String moduleName = myModuleNameEnumerator.valueOf(moduleId); + if (LOG.assertTrue(moduleName != null)) { + result.add(moduleName); + } + } + return result; + } + + void removeTest(int testId) throws IOException { + myTestNameToRunModule.remove(testId); + } + + private static class IntSeqExternalizer implements DataExternalizer { + public void save(@NotNull DataOutput dataOutput, TIntArrayList testNameIds) throws IOException { + for (int testNameId : testNameIds.toNativeArray()) DataInputOutputUtil.writeINT(dataOutput, testNameId); + } + + public TIntArrayList read(@NotNull DataInput dataInput) throws IOException { + TIntHashSet result = new TIntHashSet(); + + while (((InputStream)dataInput).available() > 0) { + int id = DataInputOutputUtil.readINT(dataInput); + result.add(id); + } + + return new TIntArrayList(result.toArray()); + } + } +} diff --git a/plugins/junit5_rt_tests/test/com/intellij/testDiscovery/TestDiscoveryJUnitIntegrationTest.java b/plugins/junit5_rt_tests/test/com/intellij/testDiscovery/TestDiscoveryJUnitIntegrationTest.java index 8832ce440ecf..03f71d3d1596 100644 --- a/plugins/junit5_rt_tests/test/com/intellij/testDiscovery/TestDiscoveryJUnitIntegrationTest.java +++ b/plugins/junit5_rt_tests/test/com/intellij/testDiscovery/TestDiscoveryJUnitIntegrationTest.java @@ -96,9 +96,18 @@ public class TestDiscoveryJUnitIntegrationTest extends JUnitAbstractIntegrationT } private void assertTestDiscoveryIndex(String className, String methodName, Pair... expectedTests) throws IOException { - MultiMap rawActualTests = TestDiscoveryIndex.getInstance(myProject).getTestsByMethodName(className, methodName, JUnitConfiguration.FRAMEWORK_ID); + TestDiscoveryIndex testDiscoveryIndex = TestDiscoveryIndex.getInstance(myProject); + MultiMap rawActualTests = testDiscoveryIndex.getTestsByMethodName(className, methodName, JUnitConfiguration.FRAMEWORK_ID); Set> actualTests = rawActualTests.entrySet().stream().flatMap(e -> e.getValue().stream().map(m -> Pair.create(e.getKey(), m))).collect(Collectors.toSet()); assertEquals(ContainerUtil.newHashSet(expectedTests), actualTests); + + Set modules = actualTests + .stream() + .flatMap( + test -> testDiscoveryIndex.getTestModulesByMethodName(test.getFirst(), test.getSecond(), JUnitConfiguration.FRAMEWORK_ID).stream()) + .collect(Collectors.toSet()); + String module = assertOneElement(modules); + assertEquals(myModule.getName(), module); } private static Pair t(String testClassName, String testMethodName) {