From 93204abcaf001f830ad361b7a34a387b10cd162f Mon Sep 17 00:00:00 2001 From: Eugene Zhuravlev Date: Mon, 10 Sep 2012 22:44:43 +0200 Subject: [PATCH] do not store paths in symbol table => honor file system's case sensitivity when comparing files --- .../dependencyView/DependencyContext.java | 14 - .../java/dependencyView/IntObjectMaplet.java | 69 +++++ .../IntObjectPersistentMaplet.java | 138 ++++++++++ .../IntObjectTransientMaplet.java | 56 ++++ .../java/dependencyView/Mappings.java | 232 +++++++--------- .../ObjectObjectMultiMaplet.java | 105 ++++++++ .../ObjectObjectPersistentMultiMaplet.java | 248 ++++++++++++++++++ .../ObjectObjectTransientMultiMaplet.java | 130 +++++++++ .../incremental/storage/BuildDataManager.java | 2 +- .../storage/FileKeyDescriptor.java | 34 +++ .../incremental/storage/TimestampStorage.java | 23 -- 11 files changed, 872 insertions(+), 179 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectMaplet.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMaplet.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectTransientMaplet.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectMultiMaplet.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectTransientMultiMaplet.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileKeyDescriptor.java diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/DependencyContext.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/DependencyContext.java index 75b842cbb7de..e374f74de44e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/DependencyContext.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/DependencyContext.java @@ -84,20 +84,6 @@ class DependencyContext { } } - public int getFilePath(final String path) { - try { - if (StringUtil.isEmpty(path)) { - return myEmptyName; - } - final String _path = FileUtil.toSystemIndependentName(path); - //return myEnumerator.enumerate(SystemInfo.isFileSystemCaseSensitive ? _path : _path.toLowerCase(Locale.US)); - return myEnumerator.enumerate(_path); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - public void close() { try { myEnumerator.close(); diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectMaplet.java new file mode 100644 index 000000000000..23782adaebf1 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectMaplet.java @@ -0,0 +1,69 @@ +/* + * Copyright 2000-2011 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jetbrains.jps.builders.java.dependencyView; + +import gnu.trove.TIntObjectProcedure; + +import java.io.PrintStream; + +/** + * Created by IntelliJ IDEA. + * User: db + * Date: 04.11.11 + * Time: 23:48 + * To change this template use File | Settings | File Templates. + */ +abstract class IntObjectMaplet implements Streamable { + abstract boolean containsKey(final int key); + + abstract V get(final int key); + + abstract void put(final int key, final V value); + + abstract void putAll(IntObjectMaplet m); + + abstract void remove(final int key); + + abstract void close(); + + abstract void forEachEntry(TIntObjectProcedure proc); + + abstract void flush(boolean memoryCachesOnly); + + public void toStream(final DependencyContext context, final PrintStream stream) { + final OrderProvider op = new OrderProvider(context); + + forEachEntry(new TIntObjectProcedure() { + @Override + public boolean execute(final int a, final V b) { + op.register(a); + return true; + } + }); + + final int[] keys = op.get(); + + for (final int a : keys) { + final V b = get(a); + + stream.print(" "); + stream.print(context.getValue(a)); + stream.print(" -> "); + + stream.print(b.toString()); + } + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMaplet.java new file mode 100644 index 000000000000..37d6d374bdb2 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMaplet.java @@ -0,0 +1,138 @@ +package org.jetbrains.jps.builders.java.dependencyView; + +import com.intellij.util.Processor; +import com.intellij.util.containers.SLRUCache; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.IntInlineKeyDescriptor; +import com.intellij.util.io.PersistentHashMap; +import gnu.trove.TIntObjectProcedure; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; + +/** + * @author Eugene Zhuravlev + * Date: 9/10/12 + */ +public class IntObjectPersistentMaplet extends IntObjectMaplet{ + + private static final Object NULL_OBJ = new Object(); + private static final int CACHE_SIZE = 512; + private final PersistentHashMap myMap; + private final SLRUCache myCache; + + public IntObjectPersistentMaplet(final File file, final DataExternalizer externalizer) { + try { + myMap = new PersistentHashMap(file, new IntInlineKeyDescriptor(), externalizer); + myCache = new SLRUCache(CACHE_SIZE, CACHE_SIZE) { + @NotNull + @Override + public Object createValue(Integer key) { + try { + final V v1 = myMap.get(key); + return v1 == null? NULL_OBJ : v1; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean containsKey(final int key) { + try { + return myMap.containsMapping(key); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public V get(final int key) { + final Object obj = myCache.get(key); + return obj == NULL_OBJ? null : (V)obj; + } + + @Override + public void put(final int key, final V value) { + try { + myCache.remove(key); + myMap.put(key, value); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void putAll(final IntObjectMaplet m) { + m.forEachEntry(new TIntObjectProcedure() { + @Override + public boolean execute(int key, V value) { + put(key, value); + return true; + } + }); + } + + @Override + public void remove(final int key) { + try { + myCache.remove(key); + myMap.remove(key); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + try { + myCache.clear(); + myMap.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void flush(boolean memoryCachesOnly) { + if (memoryCachesOnly) { + if (myMap.isDirty()) { + myMap.dropMemoryCaches(); + } + } + else { + myMap.force(); + } + } + + @Override + public void forEachEntry(final TIntObjectProcedure proc) { + try { + myMap.processKeysWithExistingMapping(new Processor() { + @Override + public boolean process(Integer key) { + try { + final V value = myMap.get(key); + return value == null? proc.execute(key, null) : proc.execute(key, value); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectTransientMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectTransientMaplet.java new file mode 100644 index 000000000000..40c13670e715 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectTransientMaplet.java @@ -0,0 +1,56 @@ +package org.jetbrains.jps.builders.java.dependencyView; + +import gnu.trove.TIntObjectHashMap; +import gnu.trove.TIntObjectProcedure; + +/** + * @author Eugene Zhuravlev + * Date: 9/10/12 + */ +public class IntObjectTransientMaplet extends IntObjectMaplet{ + private final TIntObjectHashMap myMap = new TIntObjectHashMap(); + @Override + boolean containsKey(int key) { + return myMap.containsKey(key); + } + + @Override + V get(int key) { + return myMap.get(key); + } + + @Override + void put(int key, V value) { + myMap.put(key, value); + } + + @Override + void putAll(IntObjectMaplet m) { + m.forEachEntry(new TIntObjectProcedure() { + @Override + public boolean execute(int key, V value) { + myMap.put(key, value); + return true; + } + }); + } + + @Override + void remove(int key) { + myMap.remove(key); + } + + @Override + void close() { + myMap.clear(); + } + + @Override + void forEachEntry(TIntObjectProcedure proc) { + myMap.forEachEntry(proc); + } + + @Override + void flush(boolean memoryCachesOnly) { + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java index c63c0b29f996..ffda5c7e2280 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java @@ -5,14 +5,12 @@ import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.io.IntInlineKeyDescriptor; -import gnu.trove.TIntHashSet; -import gnu.trove.TIntIntProcedure; -import gnu.trove.TIntObjectProcedure; -import gnu.trove.TIntProcedure; +import gnu.trove.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.asm4.ClassReader; import org.jetbrains.asm4.Opcodes; +import org.jetbrains.jps.incremental.storage.FileKeyDescriptor; import java.io.File; import java.io.FileNotFoundException; @@ -36,8 +34,6 @@ public class Mappings { private final static String CLASS_TO_SUBCLASSES = "classToSubclasses.tab"; private final static String CLASS_TO_CLASS = "classToClass.tab"; private final static String SOURCE_TO_CLASS = "sourceToClass.tab"; - private final static String SOURCE_TO_ANNOTATIONS = "sourceToAnnotations.tab"; - private final static String SOURCE_TO_USAGES = "sourceToUsages.tab"; private final static String CLASS_TO_SOURCE = "classToSource.tab"; private static final IntInlineKeyDescriptor INT_KEY_DESCRIPTOR = new IntInlineKeyDescriptor(); private static final int DEFAULT_SET_CAPACITY = 32; @@ -54,7 +50,7 @@ public class Mappings { private boolean myIsRebuild = false; private final TIntHashSet myChangedClasses; - private final TIntHashSet myChangedFiles; + private final THashSet myChangedFiles; private final Set myDeletedClasses; private final Object myLock; private final File myRootDir; @@ -66,8 +62,8 @@ public class Mappings { private IntIntMultiMaplet myClassToSubclasses; private IntIntMultiMaplet myClassToClassDependency; - private IntObjectMultiMaplet mySourceFileToClasses; - private IntIntMaplet myClassToSourceFile; + private ObjectObjectMultiMaplet mySourceFileToClasses; + private IntObjectMaplet myClassToSourceFile; private IntIntTransientMultiMaplet myRemovedSuperClasses; private IntIntTransientMultiMaplet myAddedSuperClasses; @@ -79,7 +75,7 @@ public class Mappings { myLock = base.myLock; myIsDelta = true; myChangedClasses = new TIntHashSet(DEFAULT_SET_CAPACITY, DEFAULT_SET_LOAD_FACTOR); - myChangedFiles = new TIntHashSet(DEFAULT_SET_CAPACITY, DEFAULT_SET_LOAD_FACTOR); + myChangedFiles = new THashSet(FileUtil.FILE_HASHING_STRATEGY); myDeletedClasses = new HashSet(DEFAULT_SET_CAPACITY, DEFAULT_SET_LOAD_FACTOR); myDeltaIsTransient = base.myDeltaIsTransient; myRootDir = new File(FileUtil.toSystemIndependentName(base.myRootDir.getAbsolutePath()) + File.separatorChar + "myDelta"); @@ -115,8 +111,8 @@ public class Mappings { if (myIsDelta && myDeltaIsTransient) { myClassToSubclasses = new IntIntTransientMultiMaplet(); myClassToClassDependency = new IntIntTransientMultiMaplet(); - mySourceFileToClasses = new IntObjectTransientMultiMaplet(ourClassSetConstructor); - myClassToSourceFile = new IntIntTransientMaplet(); + mySourceFileToClasses = new ObjectObjectTransientMultiMaplet(FileUtil.FILE_HASHING_STRATEGY, ourClassSetConstructor); + myClassToSourceFile = new IntObjectTransientMaplet(); } else { if (myIsDelta) { @@ -124,11 +120,11 @@ public class Mappings { } myClassToSubclasses = new IntIntPersistentMultiMaplet(DependencyContext.getTableFile(myRootDir, CLASS_TO_SUBCLASSES), INT_KEY_DESCRIPTOR); myClassToClassDependency = new IntIntPersistentMultiMaplet(DependencyContext.getTableFile(myRootDir, CLASS_TO_CLASS), INT_KEY_DESCRIPTOR); - mySourceFileToClasses = new IntObjectPersistentMultiMaplet( - DependencyContext.getTableFile(myRootDir, SOURCE_TO_CLASS), INT_KEY_DESCRIPTOR, ClassRepr.externalizer(myContext), + mySourceFileToClasses = new ObjectObjectPersistentMultiMaplet( + DependencyContext.getTableFile(myRootDir, SOURCE_TO_CLASS), new FileKeyDescriptor(), ClassRepr.externalizer(myContext), ourClassSetConstructor ); - myClassToSourceFile = new IntIntPersistentMaplet(DependencyContext.getTableFile(myRootDir, CLASS_TO_SOURCE), INT_KEY_DESCRIPTOR); + myClassToSourceFile = new IntObjectPersistentMaplet(DependencyContext.getTableFile(myRootDir, CLASS_TO_SOURCE), new FileKeyDescriptor()); } } @@ -146,9 +142,8 @@ public class Mappings { private void compensateRemovedContent(final Collection compiled) { if (compiled != null) { for (final File file : compiled) { - final int fileName = myContext.getFilePath(file.getPath()); - if (!mySourceFileToClasses.containsKey(fileName)) { - mySourceFileToClasses.put(fileName, new HashSet()); + if (!mySourceFileToClasses.containsKey(file)) { + mySourceFileToClasses.put(file, new HashSet()); } } } @@ -156,9 +151,9 @@ public class Mappings { @Nullable private ClassRepr getReprByName(final int name) { - final int source = myClassToSourceFile.get(name); + final File source = myClassToSourceFile.get(name); - if (source > 0) { + if (source != null) { final Collection reprs = mySourceFileToClasses.get(source); if (reprs != null) { @@ -493,8 +488,8 @@ public class Mappings { private void affectSubclasses(final int className, final Collection affectedFiles, final Collection affectedUsages, final TIntHashSet dependants, final boolean usages) { debug("Affecting subclasses of class: ", className); - final int fileName = myClassToSourceFile.get(className); - if (fileName <= 0) { + final File fileName = myClassToSourceFile.get(className); + if (fileName == null) { debug("No source file detected for class ", className); debug("End of affectSubclasses"); return; @@ -516,7 +511,7 @@ public class Mappings { if (depClasses != null) { addAll(dependants, depClasses); } - affectedFiles.add(new File(myContext.getValue(fileName))); + affectedFiles.add(fileName); final TIntHashSet directSubclasses = myClassToSubclasses.get(className); if (directSubclasses != null) { @@ -631,18 +626,17 @@ public class Mappings { } private void affectAll(final int className, final Collection affectedFiles, @Nullable final DependentFilesFilter filter) { - final int sourceFile = myClassToSourceFile.get(className); - if (sourceFile > 0) { + final File sourceFile = myClassToSourceFile.get(className); + if (sourceFile != null) { final TIntHashSet dependants = myClassToClassDependency.get(className); if (dependants != null) { dependants.forEach(new TIntProcedure() { @Override public boolean execute(int depClass) { - final int depFile = myClassToSourceFile.get(depClass); - if (depFile > 0 && depFile != sourceFile) { - final File theFile = new File(myContext.getValue(depFile)); - if (filter == null || filter.accept(theFile)) { - affectedFiles.add(theFile); + final File depFile = myClassToSourceFile.get(depClass); + if (depFile != null && !FileUtil.filesEqual(depFile, sourceFile)) { + if (filter == null || filter.accept(depFile)) { + affectedFiles.add(depFile); } } return true; @@ -702,26 +696,17 @@ public class Mappings { debug("Root class: ", owner); final TIntHashSet propagated = self.propagateFieldAccess(isField ? member.name : myEmptyName, owner); - final TIntHashSet fileNames = new TIntHashSet(propagated.size()); propagated.forEach(new TIntProcedure() { @Override public boolean execute(int className) { - final int fileName = myClassToSourceFile.get(className); - if (fileName > 0) { - fileNames.add(fileName); + final File fileName = myClassToSourceFile.get(className); + if (fileName != null) { + debug("Adding ", fileName); + affectedFiles.add(fileName); } return true; } }); - fileNames.forEach(new TIntProcedure() { - @Override - public boolean execute(int file) { - final String fileName = myContext.getValue(file); - debug("Adding ", fileName); - affectedFiles.add(new File(fileName)); - return true; - } - }); } final String packageName = ClassRepr.getPackageName(myContext.getValue(isField ? owner : member.name)); @@ -730,24 +715,14 @@ public class Mappings { debug("Package name: ", packageName); // Package-local branch - final TIntHashSet fileNames = new TIntHashSet(DEFAULT_SET_CAPACITY, DEFAULT_SET_LOAD_FACTOR); - myClassToSourceFile.forEachEntry(new TIntIntProcedure() { + myClassToSourceFile.forEachEntry(new TIntObjectProcedure() { @Override - public boolean execute(int className, int fileName) { + public boolean execute(int className, File fileName) { if (ClassRepr.getPackageName(myContext.getValue(className)).equals(packageName)) { - fileNames.add(fileName); - } - return true; - } - }); - fileNames.forEach(new TIntProcedure() { - @Override - public boolean execute(int fileName) { - final String f = myContext.getValue(fileName); - final File file = new File(f); - if (filter == null || filter.accept(file)) { - debug("Adding: ", f); - affectedFiles.add(file); + if (filter == null || filter.accept(fileName)) { + debug("Adding: ", fileName); + affectedFiles.add(fileName); + } } return true; } @@ -859,10 +834,10 @@ public class Mappings { } private class FileClasses { - final int myFileName; + final File myFileName; final Set myFileClasses; - FileClasses(int fileName, Collection fileClasses) { + FileClasses(File fileName, Collection fileClasses) { this.myFileName = fileName; this.myFileClasses = new HashSet(fileClasses); } @@ -949,7 +924,7 @@ public class Mappings { if (removed != null) { for (final String file : removed) { - final Collection classes = mySourceFileToClasses.get(myContext.getFilePath(file)); + final Collection classes = mySourceFileToClasses.get(new File(file)); if (classes != null) { for (ClassRepr c : classes) { @@ -972,7 +947,6 @@ public class Mappings { debug("Class is annotation, skipping method analysis"); return; } - final TIntHashSet affectedFiles = new TIntHashSet(DEFAULT_SET_CAPACITY, DEFAULT_SET_LOAD_FACTOR); Ref oldItRef = null; for (final MethodRepr m : added) { debug("Method: ", m.name); @@ -1037,10 +1011,10 @@ public class Mappings { if (overrides.satisfy(method) && isInheritor) { debug("Current method overrides that found"); - final int file = myClassToSourceFile.get(methodClass.name); + final File file = myClassToSourceFile.get(methodClass.name); - if (file > 0) { - affectedFiles.add(file); + if (file != null) { + myAffectedFiles.add(file); debug("Affecting file ", file); } } @@ -1074,11 +1048,11 @@ public class Mappings { public boolean execute(int subClass) { final ClassRepr r = myFuture.reprByName(subClass); if (r != null) { - final int sourceFileName = myClassToSourceFile.get(subClass); - if (sourceFileName > 0) { + final File sourceFileName = myClassToSourceFile.get(subClass); + if (sourceFileName != null) { final int outerClass = r.getOuterClassName(); if (myFuture.isMethodVisible(outerClass, m)) { - affectedFiles.add(sourceFileName); + myAffectedFiles.add(sourceFileName); debug("Affecting file due to local overriding: ", sourceFileName); } } @@ -1089,13 +1063,6 @@ public class Mappings { } } } - affectedFiles.forEach(new TIntProcedure() { - @Override - public boolean execute(int file) { - myAffectedFiles.add(new File(myContext.getValue(file))); - return true; - } - }); debug("End of added methods processing"); } @@ -1105,7 +1072,6 @@ public class Mappings { return; } debug("Processing removed methods:"); - final TIntHashSet affectedFiles = new TIntHashSet(DEFAULT_SET_CAPACITY, DEFAULT_SET_LOAD_FACTOR); for (final MethodRepr m : removed) { debug("Method ", m.name); @@ -1140,8 +1106,8 @@ public class Mappings { myFuture.addOverridingMethods(m, it, MethodRepr.equalByJavaRules(m), overridingMethods); for (final Pair p : overridingMethods) { - final int fName = myClassToSourceFile.get(p.second.name); - affectedFiles.add(fName); + final File fName = myClassToSourceFile.get(p.second.name); + myAffectedFiles.add(fName); debug("Affecting file by overriding: ", fName); } @@ -1181,10 +1147,10 @@ public class Mappings { } if (allAbstract && visited) { - final int source = myClassToSourceFile.get(p); + final File source = myClassToSourceFile.get(p); - if (source > 0) { - affectedFiles.add(source); + if (source != null) { + myAffectedFiles.add(source); debug("Removed method is not abstract & overrides some abstract method which is not then over-overriden in subclass ", p); debug("Affecting subclass source file ", source); } @@ -1196,14 +1162,6 @@ public class Mappings { }); } } - affectedFiles.forEach(new TIntProcedure() { - @Override - public boolean execute(int file) { - final String f = myContext.getValue(file); - myAffectedFiles.add(new File(f)); - return true; - } - }); debug("End of removed methods processing"); } @@ -1264,10 +1222,9 @@ public class Mappings { final ClassRepr aClass = p.getSecond(); if (aClass != MOCK_CLASS) { - final int fileName = myClassToSourceFile.get(aClass.name); - - if (fileName > 0) { - myAffectedFiles.add(new File(myContext.getValue(fileName))); + final File fileName = myClassToSourceFile.get(aClass.name); + if (fileName != null) { + myAffectedFiles.add(fileName); } } } @@ -1340,17 +1297,17 @@ public class Mappings { public boolean execute(int subClass) { final ClassRepr r = myFuture.reprByName(subClass); if (r != null) { - final int sourceFileName = myClassToSourceFile.get(subClass); - if (sourceFileName > 0) { + final File sourceFileName = myClassToSourceFile.get(subClass); + if (sourceFileName != null) { if (r.isLocal()) { debug("Affecting local subclass (introduced field can potentially hide surrounding method parameters/local variables): ", sourceFileName); - myAffectedFiles.add(new File(myContext.getValue(sourceFileName))); + myAffectedFiles.add(sourceFileName); } else { final int outerClass = r.getOuterClassName(); if (!isEmpty(outerClass) && myFuture.isFieldVisible(outerClass, f)) { debug("Affecting inner subclass (introduced field can potentially hide surrounding class fields): ", sourceFileName); - myAffectedFiles.add(new File(myContext.getValue(sourceFileName))); + myAffectedFiles.add(sourceFileName); } } } @@ -1706,9 +1663,9 @@ public class Mappings { for (final ClassRepr c : removed) { myDelta.addDeletedClass(c); - final int fileName = myClassToSourceFile.get(c.name); + final File fileName = myClassToSourceFile.get(c.name); - if (fileName > 0) { + if (fileName != null) { myDelta.myChangedFiles.add(fileName); } @@ -1740,25 +1697,15 @@ public class Mappings { final TIntHashSet depClasses = myClassToClassDependency.get(c.name); if (depClasses != null) { - final TIntHashSet fileNames = new TIntHashSet(DEFAULT_SET_CAPACITY, DEFAULT_SET_LOAD_FACTOR); depClasses.forEach(new TIntProcedure() { @Override public boolean execute(int depClass) { - final int fName = myClassToSourceFile.get(depClass); - if (fName > 0) { - fileNames.add(fName); - } - return true; - } - }); - fileNames.forEach(new TIntProcedure() { - @Override - public boolean execute(int fName) { - final String f = myContext.getValue(fName); - final File theFile = new File(f); - if (myFilter == null || myFilter.accept(theFile)) { - debug("Adding dependent file ", f); - myAffectedFiles.add(theFile); + final File fName = myClassToSourceFile.get(depClass); + if (fName != null) { + if (myFilter == null || myFilter.accept(fName)) { + debug("Adding dependent file ", fName); + myAffectedFiles.add(fName); + } } return true; } @@ -1776,12 +1723,11 @@ public class Mappings { state.myDependants.forEach(new TIntProcedure() { @Override public boolean execute(final int depClass) { - final int depFile = myClassToSourceFile.get(depClass); + final File depFile = myClassToSourceFile.get(depClass); - if (depFile > 0) { - final File theFile = new File(myContext.getValue(depFile)); + if (depFile != null) { - if (myAffectedFiles.contains(theFile) || myCompiledFiles.contains(theFile)) { + if (myAffectedFiles.contains(depFile) || myCompiledFiles.contains(depFile)) { return true; } @@ -1804,7 +1750,7 @@ public class Mappings { for (final UsageRepr.AnnotationUsage query : state.myAnnotationQuery) { if (query.satisfies(usage)) { debug("Added file due to annotation query"); - myAffectedFiles.add(theFile); + myAffectedFiles.add(depFile); return true; } @@ -1815,14 +1761,14 @@ public class Mappings { if (constraint == null) { debug("Added file with no constraints"); - myAffectedFiles.add(theFile); + myAffectedFiles.add(depFile); return true; } else { if (constraint.checkResidence(depClass)) { debug("Added file with satisfied constraint"); - myAffectedFiles.add(theFile); + myAffectedFiles.add(depFile); return true; } @@ -1850,16 +1796,16 @@ public class Mappings { processDisappearedClasses(); final List newClasses = new ArrayList(); - myDelta.mySourceFileToClasses.forEachEntry(new TIntObjectProcedure>() { + myDelta.mySourceFileToClasses.forEachEntry(new TObjectObjectProcedure>() { @Override - public boolean execute(int fileName, Collection classes) { + public boolean execute(File fileName, Collection classes) { newClasses.add(new FileClasses(fileName, classes)); return true; } }); for (final FileClasses compiledFile : newClasses) { - final int fileName = compiledFile.myFileName; + final File fileName = compiledFile.myFileName; final Set classes = compiledFile.myFileClasses; final Set pastClasses = (Set)mySourceFileToClasses.get(fileName); final DiffState state = new DiffState(Difference.make(pastClasses, classes)); @@ -1965,7 +1911,7 @@ public class Mappings { if (removed != null) { for (final String file : removed) { - final int fileName = myContext.getFilePath(file); + final File fileName = new File(file); final Set fileClasses = (Set)mySourceFileToClasses.get(fileName); if (fileClasses != null) { @@ -2019,8 +1965,8 @@ public class Mappings { delta.getChangedClasses().forEach(new TIntProcedure() { @Override public boolean execute(final int className) { - final int sourceFile = delta.myClassToSourceFile.get(className); - if (sourceFile > 0) { + final File sourceFile = delta.myClassToSourceFile.get(className); + if (sourceFile != null) { myClassToSourceFile.put(className, sourceFile); } else { @@ -2033,9 +1979,9 @@ public class Mappings { } }); - delta.getChangedFiles().forEach(new TIntProcedure() { + delta.getChangedFiles().forEach(new TObjectProcedure() { @Override - public boolean execute(final int fileName) { + public boolean execute(final File fileName) { final Collection classes = delta.mySourceFileToClasses.get(fileName); mySourceFileToClasses.replace(fileName, classes); return true; @@ -2101,16 +2047,16 @@ public class Mappings { return new Callbacks.Backend() { public void associate(final String classFileName, final String sourceFileName, final ClassReader cr) { synchronized (myLock) { - final int classFileNameS = myContext.getFilePath(classFileName); + final int classFileNameS = myContext.get(classFileName); final Pair> result = new ClassfileAnalyzer(myContext).analyze(classFileNameS, cr); final ClassRepr repr = result.first; if (repr != null) { final Set localUsages = result.second; - final int sourceFileNameS = myContext.getFilePath(sourceFileName); + final File sourceFile = new File(sourceFileName); final int className = repr.name; - myClassToSourceFile.put(className, sourceFileNameS); - mySourceFileToClasses.put(sourceFileNameS, repr); + myClassToSourceFile.put(className, sourceFile); + mySourceFileToClasses.put(sourceFile, repr); for (final int s : repr.getSupers()) { myClassToSubclasses.put(s, className); @@ -2148,8 +2094,8 @@ public class Mappings { myPostPasses.offer(new Runnable() { public void run() { final int rootClassName = myContext.get(className.replace(".", "/")); - final int fileName = myClassToSourceFile.get(rootClassName); - final ClassRepr repr = fileName > 0? getReprByName(rootClassName) : null; + final File fileName = myClassToSourceFile.get(rootClassName); + final ClassRepr repr = fileName != null? getReprByName(rootClassName) : null; for (final String i : allImports) { final int iname = myContext.get(i.replace(".", "/")); @@ -2168,7 +2114,7 @@ public class Mappings { @Nullable public Set getClasses(final String sourceFileName) { synchronized (myLock) { - return (Set)mySourceFileToClasses.get(myContext.getFilePath(sourceFileName)); + return (Set)mySourceFileToClasses.get(new File(sourceFileName)); } } @@ -2272,9 +2218,9 @@ public class Mappings { assert (myChangedClasses != null && myChangedFiles != null); myChangedClasses.add(it); - final int file = myClassToSourceFile.get(it); + final File file = myClassToSourceFile.get(it); - if (file > 0) { + if (file != null) { myChangedFiles.add(file); } } @@ -2288,7 +2234,7 @@ public class Mappings { return myChangedClasses; } - private TIntHashSet getChangedFiles() { + private THashSet getChangedFiles() { return myChangedFiles; } @@ -2300,6 +2246,10 @@ public class Mappings { myDebugS.debug(comment, s); } + private void debug(final String comment, final File f) { + debug(comment, f.getPath()); + } + private void debug(final String comment, final String s) { myDebugS.debug(comment, s); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectMultiMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectMultiMaplet.java new file mode 100644 index 000000000000..e2bd3243e39f --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectMultiMaplet.java @@ -0,0 +1,105 @@ +/* + * Copyright 2000-2011 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jetbrains.jps.builders.java.dependencyView; + +import com.intellij.openapi.util.Pair; +import gnu.trove.TObjectObjectProcedure; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.*; + +/** + * Created by IntelliJ IDEA. + * User: db + * Date: 03.11.11 + * Time: 21:01 + * To change this template use File | Settings | File Templates. + */ +abstract class ObjectObjectMultiMaplet implements Streamable { + abstract boolean containsKey(final K key); + + abstract Collection get(final K key); + + abstract void put(final K key, final V value); + + abstract void put(final K key, final Collection value); + + abstract void replace(final K key, final Collection value); + + abstract void putAll(ObjectObjectMultiMaplet m); + + abstract void replaceAll(ObjectObjectMultiMaplet m); + + abstract void remove(final K key); + + abstract void removeFrom(final K key, final V value); + + abstract void removeAll(final K key, final Collection value); + + abstract void close(); + + abstract void forEachEntry(TObjectObjectProcedure> procedure); + + abstract void flush(boolean memoryCachesOnly); + + public void toStream(final DependencyContext context, final PrintStream stream) { + + final List> keys = new ArrayList>(); + forEachEntry(new TObjectObjectProcedure>() { + @Override + public boolean execute(final K a, final Collection b) { + keys.add(new Pair(a, a.toString())); + return true; + } + }); + + Collections.sort(keys, new Comparator>() { + @Override + public int compare(Pair o1, Pair o2) { + return o1.second.compareTo(o2.second); + } + }); + + for (final Pair a: keys) { + final Collection b = get(a.first); + + stream.print(" Key: "); + stream.println(a.second); + stream.println(" Values:"); + + final List list = new LinkedList(); + + for (final V value : b) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final PrintStream s = new PrintStream(baos); + + value.toStream(context, s); + + list.add(baos.toString()); + } + + Collections.sort(list); + + for (final String l : list) { + stream.print(l); + } + + stream.println(" End Of Values"); + } + } + +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java new file mode 100644 index 000000000000..1bdd07751935 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java @@ -0,0 +1,248 @@ +package org.jetbrains.jps.builders.java.dependencyView; + +import com.intellij.util.Processor; +import com.intellij.util.containers.SLRUCache; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.PersistentHashMap; +import gnu.trove.TObjectObjectProcedure; +import org.jetbrains.annotations.NotNull; + +import java.io.*; +import java.util.Collection; +import java.util.Collections; + +/** + * @author Eugene Zhuravlev + * Date: 9/10/12 + */ +public class ObjectObjectPersistentMultiMaplet extends ObjectObjectMultiMaplet{ + private static final Collection NULL_COLLECTION = Collections.emptySet(); + private static final int CACHE_SIZE = 128; + private final PersistentHashMap> myMap; + private final DataExternalizer myValueExternalizer; + private final SLRUCache myCache; + + public ObjectObjectPersistentMultiMaplet(final File file, + final KeyDescriptor keyExternalizer, + final DataExternalizer valueExternalizer, + final CollectionFactory collectionFactory) throws IOException { + myValueExternalizer = valueExternalizer; + myMap = new PersistentHashMap>(file, keyExternalizer, new CollectionDataExternalizer(valueExternalizer, collectionFactory)); + myCache = new SLRUCache(CACHE_SIZE, CACHE_SIZE) { + @NotNull + @Override + public Collection createValue(K key) { + try { + final Collection collection = myMap.get(key); + return collection == null? NULL_COLLECTION : collection; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + } + + + @Override + public boolean containsKey(final K key) { + try { + return myMap.containsMapping(key); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Collection get(final K key) { + final Collection collection = myCache.get(key); + return collection == NULL_COLLECTION? null : collection; + } + + @Override + public void replace(K key, Collection value) { + try { + myCache.remove(key); + if (value == null || value.isEmpty()) { + myMap.remove(key); + } + else { + myMap.put(key, value); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void put(final K key, final Collection value) { + try { + myCache.remove(key); + myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { + public void append(DataOutput out) throws IOException { + for (V v : value) { + myValueExternalizer.save(out, v); + } + } + }); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void put(final K key, final V value) { + put(key, Collections.singleton(value)); + } + + @Override + public void removeAll(K key, Collection values) { + try { + final Collection collection = myCache.get(key); + + if (collection != NULL_COLLECTION) { + if (collection.removeAll(values)) { + myCache.remove(key); + if (collection.isEmpty()) { + myMap.remove(key); + } + else { + myMap.put(key, (Collection)collection); + } + } + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void removeFrom(final K key, final V value) { + try { + final Collection collection = myCache.get(key); + + if (collection != NULL_COLLECTION) { + if (collection.remove(value)) { + myCache.remove(key); + if (collection.isEmpty()) { + myMap.remove(key); + } + else { + myMap.put(key, (Collection)collection); + } + } + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void remove(final K key) { + try { + myCache.remove(key); + myMap.remove(key); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void putAll(ObjectObjectMultiMaplet m) { + m.forEachEntry(new TObjectObjectProcedure>() { + @Override + public boolean execute(K key, Collection value) { + put(key, value); + return true; + } + }); + } + + @Override + public void replaceAll(ObjectObjectMultiMaplet m) { + m.forEachEntry(new TObjectObjectProcedure>() { + @Override + public boolean execute(K key, Collection value) { + replace(key, value); + return true; + } + }); + } + + @Override + public void close() { + try { + myCache.clear(); + myMap.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void flush(boolean memoryCachesOnly) { + if (memoryCachesOnly) { + if (myMap.isDirty()) { + myMap.dropMemoryCaches(); + } + } + else { + myMap.force(); + } + } + + @Override + public void forEachEntry(final TObjectObjectProcedure> procedure) { + try { + myMap.processKeysWithExistingMapping(new Processor() { + @Override + public boolean process(K key) { + try { + return procedure.execute(key, myMap.get(key)); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static class CollectionDataExternalizer implements DataExternalizer> { + private final DataExternalizer myElementExternalizer; + private final CollectionFactory myCollectionFactory; + + public CollectionDataExternalizer(DataExternalizer elementExternalizer, + CollectionFactory collectionFactory) { + myElementExternalizer = elementExternalizer; + myCollectionFactory = collectionFactory; + } + + @Override + public void save(final DataOutput out, final Collection value) throws IOException { + for (V x : value) { + myElementExternalizer.save(out, x); + } + } + + @Override + public Collection read(final DataInput in) throws IOException { + final Collection result = myCollectionFactory.create(); + final DataInputStream stream = (DataInputStream)in; + while (stream.available() > 0) { + result.add(myElementExternalizer.read(in)); + } + return result; + } + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectTransientMultiMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectTransientMultiMaplet.java new file mode 100644 index 000000000000..dd9b8cf92033 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectTransientMultiMaplet.java @@ -0,0 +1,130 @@ +package org.jetbrains.jps.builders.java.dependencyView; + +import gnu.trove.THashMap; +import gnu.trove.TObjectHashingStrategy; +import gnu.trove.TObjectObjectProcedure; + +import java.util.Collection; + +/** + * @author Eugene Zhuravlev + * Date: 9/10/12 + */ +public class ObjectObjectTransientMultiMaplet extends ObjectObjectMultiMaplet{ + + private final THashMap> myMap; + private final CollectionFactory myCollectionFactory; + + public ObjectObjectTransientMultiMaplet(TObjectHashingStrategy hashingStrategy, CollectionFactory collectionFactory) { + myMap = new THashMap>(hashingStrategy); + myCollectionFactory = collectionFactory; + } + + @Override + public boolean containsKey(final K key) { + return myMap.containsKey(key); + } + + @Override + public Collection get(final K key) { + return myMap.get(key); + } + + @Override + public void putAll(ObjectObjectMultiMaplet m) { + m.forEachEntry(new TObjectObjectProcedure>() { + @Override + public boolean execute(K key, Collection value) { + put(key, value); + return true; + } + }); + } + + @Override + public void put(final K key, final Collection value) { + final Collection x = myMap.get(key); + if (x == null) { + myMap.put(key, value); + } + else { + x.addAll(value); + } + } + + @Override + public void replace(K key, Collection value) { + if (value == null || value.isEmpty()) { + myMap.remove(key); + } + else { + myMap.put(key, value); + } + } + + @Override + public void put(final K key, final V value) { + final Collection collection = myMap.get(key); + if (collection == null) { + final Collection x = myCollectionFactory.create(); + x.add(value); + myMap.put(key, x); + } + else { + collection.add(value); + } + } + + @Override + public void removeFrom(final K key, final V value) { + final Collection collection = myMap.get(key); + if (collection != null) { + if (collection.remove(value)) { + if (collection.isEmpty()) { + myMap.remove(key); + } + } + } + } + + @Override + public void removeAll(K key, Collection values) { + final Collection collection = myMap.get(key); + if (collection != null) { + if (collection.removeAll(values)) { + if (collection.isEmpty()) { + myMap.remove(key); + } + } + } + } + + @Override + public void remove(final K key) { + myMap.remove(key); + } + + @Override + public void replaceAll(ObjectObjectMultiMaplet m) { + m.forEachEntry(new TObjectObjectProcedure>() { + @Override + public boolean execute(K key, Collection value) { + replace(key, value); + return true; + } + }); + } + + @Override + public void forEachEntry(TObjectObjectProcedure> procedure) { + myMap.forEachEntry(procedure); + } + + @Override + public void close(){ + myMap.clear(); // free memory + } + + public void flush(boolean memoryCachesOnly) { + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index 5e72182d1c9a..47c4a251fba2 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -20,7 +20,7 @@ import java.util.Map; * Date: 10/7/11 */ public class BuildDataManager implements StorageOwner { - private static final int VERSION = 9; + private static final int VERSION = 10; private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.storage.BuildDataManager"); private static final String SRC_TO_OUTPUTS_STORAGE = "src-out"; private static final String SRC_TO_FORM_STORAGE = "src-form"; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileKeyDescriptor.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileKeyDescriptor.java new file mode 100644 index 000000000000..84953d12240d --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileKeyDescriptor.java @@ -0,0 +1,34 @@ +package org.jetbrains.jps.incremental.storage; + +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.util.io.IOUtil; +import com.intellij.util.io.KeyDescriptor; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; + +/** +* @author Eugene Zhuravlev +* Date: 9/10/12 +*/ +public final class FileKeyDescriptor implements KeyDescriptor { + private final byte[] buffer = IOUtil.allocReadWriteUTFBuffer(); + + public void save(DataOutput out, File value) throws IOException { + IOUtil.writeUTFFast(buffer, out, value.getPath()); + } + + public File read(DataInput in) throws IOException { + return new File(IOUtil.readUTFFast(buffer, in)); + } + + public int getHashCode(File value) { + return FileUtil.fileHashCode(value); + } + + public boolean isEqual(File val1, File val2) { + return FileUtil.filesEqual(val1, val2); + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/TimestampStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/TimestampStorage.java index f1493c45187f..0820f3308a7e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/TimestampStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/TimestampStorage.java @@ -1,9 +1,6 @@ package org.jetbrains.jps.incremental.storage; -import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.IOUtil; -import com.intellij.util.io.KeyDescriptor; import java.io.DataInput; import java.io.DataOutput; @@ -45,26 +42,6 @@ public class TimestampStorage extends AbstractStateStorage { - private final byte[] buffer = IOUtil.allocReadWriteUTFBuffer(); - - public void save(DataOutput out, File value) throws IOException { - IOUtil.writeUTFFast(buffer, out, value.getPath()); - } - - public File read(DataInput in) throws IOException { - return new File(IOUtil.readUTFFast(buffer, in)); - } - - public int getHashCode(File value) { - return FileUtil.fileHashCode(value); - } - - public boolean isEqual(File val1, File val2) { - return FileUtil.filesEqual(val1, val2); - } - } - private static class StateExternalizer implements DataExternalizer { public void save(DataOutput out, TimestampValidityState value) throws IOException {