diff --git a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java index 1db8875489b4..f040322f307b 100644 --- a/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java +++ b/java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java @@ -16,10 +16,9 @@ package com.intellij.execution.testDiscovery; import com.intellij.openapi.components.ProjectComponent; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.ThrowableComputable; -import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon; import com.intellij.util.io.*; import gnu.trove.THashMap; @@ -30,14 +29,13 @@ import java.io.*; import java.io.DataOutputStream; import java.util.*; import java.util.concurrent.ScheduledFuture; -import java.util.zip.Deflater; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.InflaterInputStream; /** - * Created by Maxim.Mossienko on 7/9/2015. + * @author Maxim.Mossienko on 7/9/2015. */ public class TestDiscoveryIndex implements ProjectComponent { + static final Logger LOG = Logger.getInstance(TestDiscoveryIndex.class); + private static final String REMOVED_MARKER = "-removed-"; private final Object ourLock = new Object(); private final Project myProject; @@ -98,6 +96,8 @@ public class TestDiscoveryIndex implements ProjectComponent { public void projectClosed() { } + private static final int VERSION = 1; + private final class Holder { final PersistentHashMap, Collection> myMethodQNameToTestNames; final PersistentHashMap>> myTestNameToUsedClassesAndMethodMap; @@ -108,89 +108,61 @@ public class TestDiscoveryIndex implements ProjectComponent { String path = myProject != null ? TestDiscoveryExtension.baseTestDiscoveryPathForProject(myProject) : "out"; + final File versionFile = new File(path + File.separator + "index.version"); final File methodQNameToTestNameFile = new File(path + File.separator + "methodQNameToTestName.data"); final File testNameToUsedClassesAndMethodMapFile = new File(path + File.separator + "testToCalledMethodNames"); try { - // todo better io recovery - myMethodQNameToTestNames = IOUtil.openCleanOrResetBroken( - new ThrowableComputable, Collection>, IOException>() { - @Override - public PersistentHashMap, Collection> compute() throws IOException { - return new PersistentHashMap, Collection>( - methodQNameToTestNameFile, - new StringPairKeyDescriptor(), - new DataExternalizer>() { - public void save(@NotNull DataOutput dataOutput, Collection strings) throws IOException { - for (String string : strings) IOUtil.writeUTF(dataOutput, string); - } + int version = readVersion(versionFile); + if (version != VERSION) { + LOG.info("TestDiscoveryIndex was rewritten due to version change"); + IOUtil.deleteAllFilesStartingWith(methodQNameToTestNameFile); + IOUtil.deleteAllFilesStartingWith(testNameToUsedClassesAndMethodMapFile); - public Collection read(@NotNull DataInput dataInput) throws IOException { - Set result = new THashSet(); + writeVersion(versionFile); + } - while (((InputStream)dataInput).available() > 0) { - String string = IOUtil.readUTF(dataInput); - if (REMOVED_MARKER.equals(string)) { - string = IOUtil.readUTF(dataInput); - result.remove(string); - } - else { - result.add(string); - } - } + PersistentHashMap, Collection> methodQNameToTestNames = null; + PersistentHashMap>> testNameToUsedClassesAndMethodMap = null; - return result; - } - } - ); - } - }, methodQNameToTestNameFile); - myTestNameToUsedClassesAndMethodMap = IOUtil.openCleanOrResetBroken( - new ThrowableComputable>>, IOException>() { - @Override - public PersistentHashMap>> compute() throws IOException { - return new PersistentHashMap>>( - testNameToUsedClassesAndMethodMapFile, - EnumeratorStringDescriptor.INSTANCE, - new DataExternalizer>>() { - public void save(@NotNull DataOutput dataOutput, Map> classAndMethodsMap) - throws IOException { - BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(); - Deflater deflater = new Deflater(1); - DataOutputStream dataOutputStream = - new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(out, deflater))); - DataInputOutputUtil.writeINT(dataOutputStream, classAndMethodsMap.size()); - for (Map.Entry> e : classAndMethodsMap.entrySet()) { - IOUtil.writeUTF(dataOutputStream, e.getKey()); - DataInputOutputUtil.writeINT(dataOutputStream, e.getValue().size()); - for (String methodName : e.getValue()) IOUtil.writeUTF(dataOutputStream, methodName); - } - dataOutputStream.close(); - deflater.end(); - dataOutput.write(out.getInternalBuffer(), 0, out.size()); - } + for(int i = 0; i < 2; ++i) { + try { + methodQNameToTestNames = new PersistentHashMap, Collection>( + methodQNameToTestNameFile, + new StringPairKeyDescriptor(), + new TestNamesExternalizer() + ); + } catch (Throwable throwable) { + LOG.info("TestDiscoveryIndex problem", throwable); + IOUtil.deleteAllFilesStartingWith(methodQNameToTestNameFile); + IOUtil.deleteAllFilesStartingWith(testNameToUsedClassesAndMethodMapFile); + continue; // try another time + } + try { + testNameToUsedClassesAndMethodMap = new PersistentHashMap>>( + testNameToUsedClassesAndMethodMapFile, + EnumeratorStringDescriptor.INSTANCE, + new ClassesAndMethodsMapDataExternalizer() + ); + } catch (Throwable throwable) { + LOG.info("TestDiscoveryIndex problem2", throwable); + try { + methodQNameToTestNames.close(); + methodQNameToTestNames = null; + } catch (Throwable ignore) {} + IOUtil.deleteAllFilesStartingWith(methodQNameToTestNameFile); + IOUtil.deleteAllFilesStartingWith(testNameToUsedClassesAndMethodMapFile); + // try another time + } + + if (testNameToUsedClassesAndMethodMap != null) break; + } + + assert testNameToUsedClassesAndMethodMap != null; + assert methodQNameToTestNameFile != null; + myMethodQNameToTestNames = methodQNameToTestNames; + myTestNameToUsedClassesAndMethodMap = testNameToUsedClassesAndMethodMap; - public Map> read(@NotNull DataInput dataInput) throws IOException { - byte[] buf; - dataInput.readFully(buf = new byte[((InputStream)dataInput).available()]); - DataInputStream dataInputStream = - new DataInputStream(new InflaterInputStream(new ByteArrayInputStream(buf))); - int numberOfClasses = DataInputOutputUtil.readINT(dataInputStream); - THashMap> result = new THashMap>(numberOfClasses); - while (numberOfClasses-- > 0) { - String className = IOUtil.readUTF(dataInputStream); - int numberOfMethods = DataInputOutputUtil.readINT(dataInputStream); - ArrayList methods = new ArrayList(numberOfMethods); - while (numberOfMethods-- > 0) methods.add(IOUtil.readUTF(dataInputStream)); - result.put(className, methods); - } - dataInputStream.close(); - return result; - } - } - ); - } - }, testNameToUsedClassesAndMethodMapFile); myFlushingFuture = FlushingDaemon.everyFiveSeconds(new Runnable() { @Override public void run() { @@ -214,6 +186,28 @@ public class TestDiscoveryIndex implements ProjectComponent { } } + private void writeVersion(File versionFile) throws IOException { + final DataOutputStream versionOut = new DataOutputStream(new FileOutputStream(versionFile)); + + try { + DataInputOutputUtil.writeINT(versionOut, VERSION); + } finally { + try { versionOut.close(); } catch (IOException ignore) {} + } + } + + private int readVersion(File versionFile) throws IOException { + if (!versionFile.exists()) return 0; + final DataInputStream versionInput = new DataInputStream(new FileInputStream(versionFile)); + int version; + try { + version = DataInputOutputUtil.readINT(versionInput); + } finally { + try { versionInput.close(); } catch (IOException ignore) {} + } + return version; + } + void dispose() { assert Thread.holdsLock(ourLock); try { @@ -228,7 +222,53 @@ public class TestDiscoveryIndex implements ProjectComponent { } } - + private class TestNamesExternalizer implements DataExternalizer> { + public void save(@NotNull DataOutput dataOutput, Collection strings) throws IOException { + for (String string : strings) IOUtil.writeUTF(dataOutput, string); + } + + public Collection read(@NotNull DataInput dataInput) throws IOException { + Set result = new THashSet(); + + while (((InputStream)dataInput).available() > 0) { + String string = IOUtil.readUTF(dataInput); + if (REMOVED_MARKER.equals(string)) { + string = IOUtil.readUTF(dataInput); + result.remove(string); + } + else { + result.add(string); + } + } + + return result; + } + } + + private class ClassesAndMethodsMapDataExternalizer implements DataExternalizer>> { + public void save(@NotNull DataOutput dataOutput, Map> classAndMethodsMap) + throws IOException { + DataInputOutputUtil.writeINT(dataOutput, classAndMethodsMap.size()); + for (Map.Entry> e : classAndMethodsMap.entrySet()) { + IOUtil.writeUTF(dataOutput, e.getKey()); + DataInputOutputUtil.writeINT(dataOutput, e.getValue().size()); + for (String methodName : e.getValue()) IOUtil.writeUTF(dataOutput, methodName); + } + } + + public Map> read(@NotNull DataInput dataInput) throws IOException { + int numberOfClasses = DataInputOutputUtil.readINT(dataInput); + THashMap> result = new THashMap>(numberOfClasses); + while (numberOfClasses-- > 0) { + String className = IOUtil.readUTF(dataInput); + int numberOfMethods = DataInputOutputUtil.readINT(dataInput); + ArrayList methods = new ArrayList(numberOfMethods); + while (numberOfMethods-- > 0) methods.add(IOUtil.readUTF(dataInput)); + result.put(className, methods); + } + return result; + } + } } private static class StringPairKeyDescriptor implements KeyDescriptor> { @@ -257,10 +297,13 @@ public class TestDiscoveryIndex implements ProjectComponent { } public void updateFromTestTrace(File file) throws IOException { - synchronized (ourLock) { - int fileNameDotIndex = file.getName().lastIndexOf('.'); - final String testName = fileNameDotIndex != -1 ? file.getName().substring(0, fileNameDotIndex) : file.getName(); + int fileNameDotIndex = file.getName().lastIndexOf('.'); + final String testName = fileNameDotIndex != -1 ? file.getName().substring(0, fileNameDotIndex) : file.getName(); + doUpdateFromTestTrace(file, testName); + } + private void doUpdateFromTestTrace(File file, final String testName) throws IOException { + synchronized (ourLock) { Holder holder = getHolder(); if (holder.myDisposed) return; Map> classData = loadClassAndMethodsMap(file); @@ -268,7 +311,7 @@ public class TestDiscoveryIndex implements ProjectComponent { ValueDiff valueDiff = new ValueDiff(classData, previousClassData); - if (valueDiff.myRemovedClassData != null && !valueDiff.myRemovedClassData.isEmpty()) { + if (valueDiff.hasRemovedDelta()) { for (String classQName : valueDiff.myRemovedClassData.keySet()) { for (String methodName : valueDiff.myRemovedClassData.get(classQName)) { holder.myMethodQNameToTestNames.appendData(createKey(classQName, methodName), @@ -284,7 +327,7 @@ public class TestDiscoveryIndex implements ProjectComponent { } } - if (valueDiff.myAddedOrChangedClassData != null && !valueDiff.myAddedOrChangedClassData.isEmpty()) { + if (valueDiff.hasAddedDelta()) { for (String classQName : valueDiff.myAddedOrChangedClassData.keySet()) { for (String methodName : valueDiff.myAddedOrChangedClassData.get(classQName)) { holder.myMethodQNameToTestNames.appendData(createKey(classQName, methodName), @@ -298,9 +341,7 @@ public class TestDiscoveryIndex implements ProjectComponent { } } - if (valueDiff.myAddedOrChangedClassData != null && !valueDiff.myAddedOrChangedClassData.isEmpty() || - valueDiff.myRemovedClassData != null && !valueDiff.myRemovedClassData.isEmpty() - ) { + if (valueDiff.hasAddedDelta() || valueDiff.hasRemovedDelta()) { holder.myTestNameToUsedClassesAndMethodMap.put(testName, classData); } } @@ -354,24 +395,33 @@ public class TestDiscoveryIndex implements ProjectComponent { myAddedOrChangedClassData = addedOrChangedClassData; myRemovedClassData = removedClassData; } + + public boolean hasRemovedDelta() { + return myRemovedClassData != null && !myRemovedClassData.isEmpty(); + } + + public boolean hasAddedDelta() { + return myAddedOrChangedClassData != null && !myAddedOrChangedClassData.isEmpty(); + } } @NotNull private static Map> loadClassAndMethodsMap(File file) throws IOException { - DataInputStream inputStream = new DataInputStream(new InflaterInputStream(new BufferedInputStream(new FileInputStream(file)))); + DataInputStream inputStream = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 64 * 1024)); + byte[] buffer = IOUtil.allocReadWriteUTFBuffer(); try { - int numberOfClasses = inputStream.readInt(); - Map> classData = new THashMap>(); + int numberOfClasses = DataInputOutputUtil.readINT(inputStream); + Map> classData = new THashMap>(numberOfClasses, 0.5f); while (numberOfClasses-- > 0) { - String classQName = inputStream.readUTF(); - int numberOfMethods = inputStream.readInt(); + String classQName = IOUtil.readUTFFast(buffer, inputStream); + int numberOfMethods = DataInputOutputUtil.readINT(inputStream); List methodsList; classData.put(classQName, methodsList = new ArrayList(numberOfMethods)); //System.out.println(classQName + "," + numberOfMethods); while (numberOfMethods-- > 0) { - String methodName = inputStream.readUTF(); + String methodName = IOUtil.readUTFFast(buffer, inputStream); methodsList.add(methodName); //System.out.println(methodName); } diff --git a/java/execution/impl/testDiscovery/org/jetbrains/testme/instrumentation/ProjectData.java b/java/execution/impl/testDiscovery/org/jetbrains/testme/instrumentation/ProjectData.java index 31336bd6e1fc..b05a9cfcedca 100644 --- a/java/execution/impl/testDiscovery/org/jetbrains/testme/instrumentation/ProjectData.java +++ b/java/execution/impl/testDiscovery/org/jetbrains/testme/instrumentation/ProjectData.java @@ -4,8 +4,6 @@ import java.io.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.zip.Deflater; -import java.util.zip.DeflaterOutputStream; public class ProjectData { public static final String PROJECT_DATA_OWNER = "org/jetbrains/testme/instrumentation/ProjectData"; @@ -57,6 +55,48 @@ public class ProjectData { myTrace3.put(className, methodNames); } + private static final int STRING_LENGTH_THRESHOLD = 255; + private static final int STRING_HEADER_SIZE = 1; + + private static void writeUTF(DataOutput storage, byte[] buffer, String value) throws IOException { + int len = value.length(); + + if (len < STRING_LENGTH_THRESHOLD) { + buffer[0] = (byte)len; + boolean isAscii = true; + for (int i = 0; i < len; i++) { + char c = value.charAt(i); + if (c >= 128) { + isAscii = false; + break; + } + buffer[i + STRING_HEADER_SIZE] = (byte)c; + } + if (isAscii) { + storage.write(buffer, 0, len + STRING_HEADER_SIZE); + return; + } + } + + storage.writeByte((byte)0xFF); + storage.writeUTF(value); + } + + private static void writeINT(DataOutput record, int val) throws IOException { + if (0 <= val && val < 192) { + record.writeByte(val); + } + else { + record.writeByte(192 + (val & 0x3F)); + val >>>= 6; + while (val >= 128) { + record.writeByte((val & 0x7F) | 0x80); + val >>>= 7; + } + record.writeByte(val); + } + } + private static volatile boolean traceDirDumped; public synchronized void testEnded(final String name) { @@ -74,9 +114,9 @@ public class ProjectData { traceFile.createNewFile(); } DataOutputStream os = null; - Deflater def = new Deflater(1); try { - os = new DataOutputStream(new DeflaterOutputStream(new BufferedOutputStream(new FileOutputStream(traceFile)), def)); + os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(traceFile), 64 * 1024)); + final byte[] buffer = new byte[STRING_LENGTH_THRESHOLD + STRING_HEADER_SIZE]; //saveOldTrace(os); @@ -94,7 +134,7 @@ public class ProjectData { } } - os.writeInt(classToUsedMethods.size()); + writeINT(os, classToUsedMethods.size()); for(Map.Entry e: myTrace2.entrySet()) { final boolean[] used = e.getValue(); final String className = e.getKey(); @@ -104,13 +144,13 @@ public class ProjectData { int usedMethodsCount = integer; - os.writeUTF(className); - os.writeInt(usedMethodsCount); + writeUTF(os, buffer, className); + writeINT(os, usedMethodsCount); String[] methodNames = myTrace3.get(className); for (int i = 0, len = used.length; i < len; ++i) { - // we check usedMethodCount here since used was observed to change // ? - if (used[i] && usedMethodsCount-- > 0) os.writeUTF(methodNames[i]); + // we check usedMethodCount here since used can still be updated by other threads + if (used[i] && usedMethodsCount-- > 0) writeUTF(os, buffer, methodNames[i]); } } } @@ -118,7 +158,6 @@ public class ProjectData { if (os != null) { os.close(); } - def.end(); } } catch (IOException e) {