MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

- pass the coverage data without zipping with large outputbuffer and better int encoding

Browse Source 
- versioning for TestDiscoveryIndex + better broken data handling + support new data format
This commit is contained in:
Maxim.Mossienko
2015-07-22 20:21:49 +02:00
parent d865256e01
commit b0083373dc
2 changed files with 195 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/execution/impl/src/com/intellij/execution/testDiscovery/TestDiscoveryIndex.java
+146 -96
View File
@@ -16,10 +16,9 @@
package com.intellij.execution.testDiscovery;
import com.intellij.openapi.components.ProjectComponent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.ThrowableComputable;
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon;
import com.intellij.util.io.*;
import gnu.trove.THashMap;
@@ -30,14 +29,13 @@ import java.io.*;
import java.io.DataOutputStream;
import java.util.*;
import java.util.concurrent.ScheduledFuture;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
/**
* Created by Maxim.Mossienko on 7/9/2015.
* @author Maxim.Mossienko on 7/9/2015.
*/
public class TestDiscoveryIndex implements ProjectComponent {
static final Logger LOG = Logger.getInstance(TestDiscoveryIndex.class);
private static final String REMOVED_MARKER = "-removed-";
private final Object ourLock = new Object();
private final Project myProject;
@@ -98,6 +96,8 @@ public class TestDiscoveryIndex implements ProjectComponent {
public void projectClosed() {
}
private static final int VERSION = 1;
private final class Holder {
final PersistentHashMap<Pair<String, String>, Collection<String>> myMethodQNameToTestNames;
final PersistentHashMap<String, Map<String, List<String>>> myTestNameToUsedClassesAndMethodMap;
@@ -108,89 +108,61 @@ public class TestDiscoveryIndex implements ProjectComponent {
String path = myProject != null ?
TestDiscoveryExtension.baseTestDiscoveryPathForProject(myProject) :
"out";
final File versionFile = new File(path + File.separator + "index.version");
final File methodQNameToTestNameFile = new File(path + File.separator + "methodQNameToTestName.data");
final File testNameToUsedClassesAndMethodMapFile = new File(path + File.separator + "testToCalledMethodNames");
try {
// todo better io recovery
myMethodQNameToTestNames = IOUtil.openCleanOrResetBroken(
new ThrowableComputable<PersistentHashMap<Pair<String, String>, Collection<String>>, IOException>() {
@Override
public PersistentHashMap<Pair<String, String>, Collection<String>> compute() throws IOException {
return new PersistentHashMap<Pair<String, String>, Collection<String>>(
methodQNameToTestNameFile,
new StringPairKeyDescriptor(),
new DataExternalizer<Collection<String>>() {
public void save(@NotNull DataOutput dataOutput, Collection<String> strings) throws IOException {
for (String string : strings) IOUtil.writeUTF(dataOutput, string);
}
int version = readVersion(versionFile);
if (version != VERSION) {
LOG.info("TestDiscoveryIndex was rewritten due to version change");
IOUtil.deleteAllFilesStartingWith(methodQNameToTestNameFile);
IOUtil.deleteAllFilesStartingWith(testNameToUsedClassesAndMethodMapFile);
public Collection<String> read(@NotNull DataInput dataInput) throws IOException {
Set<String> result = new THashSet<String>();
writeVersion(versionFile);
}
while (((InputStream)dataInput).available() > 0) {
String string = IOUtil.readUTF(dataInput);
if (REMOVED_MARKER.equals(string)) {
string = IOUtil.readUTF(dataInput);
result.remove(string);
}
else {
result.add(string);
}
}
PersistentHashMap<Pair<String, String>, Collection<String>> methodQNameToTestNames = null;
PersistentHashMap<String, Map<String, List<String>>> testNameToUsedClassesAndMethodMap = null;
return result;
}
}
);
}
}, methodQNameToTestNameFile);
myTestNameToUsedClassesAndMethodMap = IOUtil.openCleanOrResetBroken(
new ThrowableComputable<PersistentHashMap<String, Map<String, List<String>>>, IOException>() {
@Override
public PersistentHashMap<String, Map<String, List<String>>> compute() throws IOException {
return new PersistentHashMap<String, Map<String, List<String>>>(
testNameToUsedClassesAndMethodMapFile,
EnumeratorStringDescriptor.INSTANCE,
new DataExternalizer<Map<String, List<String>>>() {
public void save(@NotNull DataOutput dataOutput, Map<String, List<String>> classAndMethodsMap)
throws IOException {
BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream();
Deflater deflater = new Deflater(1);
DataOutputStream dataOutputStream =
new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(out, deflater)));
DataInputOutputUtil.writeINT(dataOutputStream, classAndMethodsMap.size());
for (Map.Entry<String, List<String>> e : classAndMethodsMap.entrySet()) {
IOUtil.writeUTF(dataOutputStream, e.getKey());
DataInputOutputUtil.writeINT(dataOutputStream, e.getValue().size());
for (String methodName : e.getValue()) IOUtil.writeUTF(dataOutputStream, methodName);
}
dataOutputStream.close();
deflater.end();
dataOutput.write(out.getInternalBuffer(), 0, out.size());
}
for(int i = 0; i < 2; ++i) {
try {
methodQNameToTestNames = new PersistentHashMap<Pair<String, String>, Collection<String>>(
methodQNameToTestNameFile,
new StringPairKeyDescriptor(),
new TestNamesExternalizer()
);
} catch (Throwable throwable) {
LOG.info("TestDiscoveryIndex problem", throwable);
IOUtil.deleteAllFilesStartingWith(methodQNameToTestNameFile);
IOUtil.deleteAllFilesStartingWith(testNameToUsedClassesAndMethodMapFile);
continue; // try another time
}
try {
testNameToUsedClassesAndMethodMap = new PersistentHashMap<String, Map<String, List<String>>>(
testNameToUsedClassesAndMethodMapFile,
EnumeratorStringDescriptor.INSTANCE,
new ClassesAndMethodsMapDataExternalizer()
);
} catch (Throwable throwable) {
LOG.info("TestDiscoveryIndex problem2", throwable);
try {
methodQNameToTestNames.close();
methodQNameToTestNames = null;
} catch (Throwable ignore) {}
IOUtil.deleteAllFilesStartingWith(methodQNameToTestNameFile);
IOUtil.deleteAllFilesStartingWith(testNameToUsedClassesAndMethodMapFile);
// try another time
}
if (testNameToUsedClassesAndMethodMap != null) break;
}
assert testNameToUsedClassesAndMethodMap != null;
assert methodQNameToTestNameFile != null;
myMethodQNameToTestNames = methodQNameToTestNames;
myTestNameToUsedClassesAndMethodMap = testNameToUsedClassesAndMethodMap;
public Map<String, List<String>> read(@NotNull DataInput dataInput) throws IOException {
byte[] buf;
dataInput.readFully(buf = new byte[((InputStream)dataInput).available()]);
DataInputStream dataInputStream =
new DataInputStream(new InflaterInputStream(new ByteArrayInputStream(buf)));
int numberOfClasses = DataInputOutputUtil.readINT(dataInputStream);
THashMap<String, List<String>> result = new THashMap<String, List<String>>(numberOfClasses);
while (numberOfClasses-- > 0) {
String className = IOUtil.readUTF(dataInputStream);
int numberOfMethods = DataInputOutputUtil.readINT(dataInputStream);
ArrayList<String> methods = new ArrayList<String>(numberOfMethods);
while (numberOfMethods-- > 0) methods.add(IOUtil.readUTF(dataInputStream));
result.put(className, methods);
}
dataInputStream.close();
return result;
}
}
);
}
}, testNameToUsedClassesAndMethodMapFile);
myFlushingFuture = FlushingDaemon.everyFiveSeconds(new Runnable() {
@Override
public void run() {
@@ -214,6 +186,28 @@ public class TestDiscoveryIndex implements ProjectComponent {
}
}
private void writeVersion(File versionFile) throws IOException {
final DataOutputStream versionOut = new DataOutputStream(new FileOutputStream(versionFile));
try {
DataInputOutputUtil.writeINT(versionOut, VERSION);
} finally {
try { versionOut.close(); } catch (IOException ignore) {}
}
}
private int readVersion(File versionFile) throws IOException {
if (!versionFile.exists()) return 0;
final DataInputStream versionInput = new DataInputStream(new FileInputStream(versionFile));
int version;
try {
version = DataInputOutputUtil.readINT(versionInput);
} finally {
try { versionInput.close(); } catch (IOException ignore) {}
}
return version;
}
void dispose() {
assert Thread.holdsLock(ourLock);
try {
@@ -228,7 +222,53 @@ public class TestDiscoveryIndex implements ProjectComponent {
}
}
private class TestNamesExternalizer implements DataExternalizer<Collection<String>> {
public void save(@NotNull DataOutput dataOutput, Collection<String> strings) throws IOException {
for (String string : strings) IOUtil.writeUTF(dataOutput, string);
}
public Collection<String> read(@NotNull DataInput dataInput) throws IOException {
Set<String> result = new THashSet<String>();
while (((InputStream)dataInput).available() > 0) {
String string = IOUtil.readUTF(dataInput);
if (REMOVED_MARKER.equals(string)) {
string = IOUtil.readUTF(dataInput);
result.remove(string);
}
else {
result.add(string);
}
}
return result;
}
}
private class ClassesAndMethodsMapDataExternalizer implements DataExternalizer<Map<String, List<String>>> {
public void save(@NotNull DataOutput dataOutput, Map<String, List<String>> classAndMethodsMap)
throws IOException {
DataInputOutputUtil.writeINT(dataOutput, classAndMethodsMap.size());
for (Map.Entry<String, List<String>> e : classAndMethodsMap.entrySet()) {
IOUtil.writeUTF(dataOutput, e.getKey());
DataInputOutputUtil.writeINT(dataOutput, e.getValue().size());
for (String methodName : e.getValue()) IOUtil.writeUTF(dataOutput, methodName);
}
}
public Map<String, List<String>> read(@NotNull DataInput dataInput) throws IOException {
int numberOfClasses = DataInputOutputUtil.readINT(dataInput);
THashMap<String, List<String>> result = new THashMap<String, List<String>>(numberOfClasses);
while (numberOfClasses-- > 0) {
String className = IOUtil.readUTF(dataInput);
int numberOfMethods = DataInputOutputUtil.readINT(dataInput);
ArrayList<String> methods = new ArrayList<String>(numberOfMethods);
while (numberOfMethods-- > 0) methods.add(IOUtil.readUTF(dataInput));
result.put(className, methods);
}
return result;
}
}
}
private static class StringPairKeyDescriptor implements KeyDescriptor<Pair<String, String>> {
@@ -257,10 +297,13 @@ public class TestDiscoveryIndex implements ProjectComponent {
}
public void updateFromTestTrace(File file) throws IOException {
synchronized (ourLock) {
int fileNameDotIndex = file.getName().lastIndexOf('.');
final String testName = fileNameDotIndex != -1 ? file.getName().substring(0, fileNameDotIndex) : file.getName();
int fileNameDotIndex = file.getName().lastIndexOf('.');
final String testName = fileNameDotIndex != -1 ? file.getName().substring(0, fileNameDotIndex) : file.getName();
doUpdateFromTestTrace(file, testName);
}
private void doUpdateFromTestTrace(File file, final String testName) throws IOException {
synchronized (ourLock) {
Holder holder = getHolder();
if (holder.myDisposed) return;
Map<String, List<String>> classData = loadClassAndMethodsMap(file);
@@ -268,7 +311,7 @@ public class TestDiscoveryIndex implements ProjectComponent {
ValueDiff valueDiff = new ValueDiff(classData, previousClassData);
if (valueDiff.myRemovedClassData != null && !valueDiff.myRemovedClassData.isEmpty()) {
if (valueDiff.hasRemovedDelta()) {
for (String classQName : valueDiff.myRemovedClassData.keySet()) {
for (String methodName : valueDiff.myRemovedClassData.get(classQName)) {
holder.myMethodQNameToTestNames.appendData(createKey(classQName, methodName),
@@ -284,7 +327,7 @@ public class TestDiscoveryIndex implements ProjectComponent {
}
}
if (valueDiff.myAddedOrChangedClassData != null && !valueDiff.myAddedOrChangedClassData.isEmpty()) {
if (valueDiff.hasAddedDelta()) {
for (String classQName : valueDiff.myAddedOrChangedClassData.keySet()) {
for (String methodName : valueDiff.myAddedOrChangedClassData.get(classQName)) {
holder.myMethodQNameToTestNames.appendData(createKey(classQName, methodName),
@@ -298,9 +341,7 @@ public class TestDiscoveryIndex implements ProjectComponent {
}
}
if (valueDiff.myAddedOrChangedClassData != null && !valueDiff.myAddedOrChangedClassData.isEmpty() ||
valueDiff.myRemovedClassData != null && !valueDiff.myRemovedClassData.isEmpty()
) {
if (valueDiff.hasAddedDelta() || valueDiff.hasRemovedDelta()) {
holder.myTestNameToUsedClassesAndMethodMap.put(testName, classData);
}
}
@@ -354,24 +395,33 @@ public class TestDiscoveryIndex implements ProjectComponent {
myAddedOrChangedClassData = addedOrChangedClassData;
myRemovedClassData = removedClassData;
}
public boolean hasRemovedDelta() {
return myRemovedClassData != null && !myRemovedClassData.isEmpty();
}
public boolean hasAddedDelta() {
return myAddedOrChangedClassData != null && !myAddedOrChangedClassData.isEmpty();
}
}
@NotNull
private static Map<String, List<String>> loadClassAndMethodsMap(File file) throws IOException {
DataInputStream inputStream = new DataInputStream(new InflaterInputStream(new BufferedInputStream(new FileInputStream(file))));
DataInputStream inputStream = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 64 * 1024));
byte[] buffer = IOUtil.allocReadWriteUTFBuffer();
try {
int numberOfClasses = inputStream.readInt();
Map<String, List<String>> classData = new THashMap<String, List<String>>();
int numberOfClasses = DataInputOutputUtil.readINT(inputStream);
Map<String, List<String>> classData = new THashMap<String, List<String>>(numberOfClasses, 0.5f);
while (numberOfClasses-- > 0) {
String classQName = inputStream.readUTF();
int numberOfMethods = inputStream.readInt();
String classQName = IOUtil.readUTFFast(buffer, inputStream);
int numberOfMethods = DataInputOutputUtil.readINT(inputStream);
List<String> methodsList;
classData.put(classQName, methodsList = new ArrayList<String>(numberOfMethods));
//System.out.println(classQName + "," + numberOfMethods);
while (numberOfMethods-- > 0) {
String methodName = inputStream.readUTF();
String methodName = IOUtil.readUTFFast(buffer, inputStream);
methodsList.add(methodName);
//System.out.println(methodName);
}
java/execution/impl/testDiscovery/org/jetbrains/testme/instrumentation/ProjectData.java
+49 -10
View File
@@ -4,8 +4,6 @@ import java.io.*;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
public class ProjectData {
public static final String PROJECT_DATA_OWNER = "org/jetbrains/testme/instrumentation/ProjectData";
@@ -57,6 +55,48 @@ public class ProjectData {
myTrace3.put(className, methodNames);
}
private static final int STRING_LENGTH_THRESHOLD = 255;
private static final int STRING_HEADER_SIZE = 1;
private static void writeUTF(DataOutput storage, byte[] buffer, String value) throws IOException {
int len = value.length();
if (len < STRING_LENGTH_THRESHOLD) {
buffer[0] = (byte)len;
boolean isAscii = true;
for (int i = 0; i < len; i++) {
char c = value.charAt(i);
if (c >= 128) {
isAscii = false;
break;
}
buffer[i + STRING_HEADER_SIZE] = (byte)c;
}
if (isAscii) {
storage.write(buffer, 0, len + STRING_HEADER_SIZE);
return;
}
}
storage.writeByte((byte)0xFF);
storage.writeUTF(value);
}
private static void writeINT(DataOutput record, int val) throws IOException {
if (0 <= val && val < 192) {
record.writeByte(val);
}
else {
record.writeByte(192 + (val & 0x3F));
val >>>= 6;
while (val >= 128) {
record.writeByte((val & 0x7F) | 0x80);
val >>>= 7;
}
record.writeByte(val);
}
}
private static volatile boolean traceDirDumped;
public synchronized void testEnded(final String name) {
@@ -74,9 +114,9 @@ public class ProjectData {
traceFile.createNewFile();
}
DataOutputStream os = null;
Deflater def = new Deflater(1);
try {
os = new DataOutputStream(new DeflaterOutputStream(new BufferedOutputStream(new FileOutputStream(traceFile)), def));
os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(traceFile), 64 * 1024));
final byte[] buffer = new byte[STRING_LENGTH_THRESHOLD + STRING_HEADER_SIZE];
//saveOldTrace(os);
@@ -94,7 +134,7 @@ public class ProjectData {
}
}
os.writeInt(classToUsedMethods.size());
writeINT(os, classToUsedMethods.size());
for(Map.Entry<String, boolean[]> e: myTrace2.entrySet()) {
final boolean[] used = e.getValue();
final String className = e.getKey();
@@ -104,13 +144,13 @@ public class ProjectData {
int usedMethodsCount = integer;
os.writeUTF(className);
os.writeInt(usedMethodsCount);
writeUTF(os, buffer, className);
writeINT(os, usedMethodsCount);
String[] methodNames = myTrace3.get(className);
for (int i = 0, len = used.length; i < len; ++i) {
// we check usedMethodCount here since used was observed to change // ?
if (used[i] && usedMethodsCount-- > 0) os.writeUTF(methodNames[i]);
// we check usedMethodCount here since used can still be updated by other threads
if (used[i] && usedMethodsCount-- > 0) writeUTF(os, buffer, methodNames[i]);
}
}
}
@@ -118,7 +158,6 @@ public class ProjectData {
if (os != null) {
os.close();
}
def.end();
}
}
catch (IOException e) {