diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java index ad76a809da90..9929c1c586d2 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java @@ -91,7 +91,7 @@ public class IdIndex extends FileBasedIndexExtension { @Override public int getVersion() { - return 11 + (ourSnapshotMappingsEnabled ? 0xFF:0); // TODO: version should enumerate all word scanner versions and build version upon that set + return 12 + (ourSnapshotMappingsEnabled ? 0xFF:0); // TODO: version should enumerate all word scanner versions and build version upon that set } @Override diff --git a/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java b/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java index 72f5f947d0e7..9025357b4908 100644 --- a/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java +++ b/platform/lang-impl/src/com/intellij/find/ngrams/TrigramIndex.java @@ -103,7 +103,7 @@ public class TrigramIndex extends ScalarIndexExtension implements Custo @Override public int getVersion() { - return ENABLED ? 2 + (IdIndex.ourSnapshotMappingsEnabled ? 0xFF:0) : 1; + return ENABLED ? 3 + (IdIndex.ourSnapshotMappingsEnabled ? 0xFF:0) : 1; } @Override diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 6cb43283952f..fc17bf25a33f 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -135,7 +135,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe LOG.info("Version has changed for stub index " + extension.getKey() + ". The index will be rebuilt."); } FileUtil.delete(indexRootDir); - IndexingStamp.rewriteVersion(versionFile, version); + IndexingStamp.rewriteVersion(versionFile, version); // todo snapshots indices } for (int attempt = 0; attempt < 2; attempt++) { @@ -157,7 +157,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe LOG.info(e); needRebuild = true; FileUtil.delete(indexRootDir); - IndexingStamp.rewriteVersion(versionFile, version); + IndexingStamp.rewriteVersion(versionFile, version); // todo snapshots indices } } return needRebuild; @@ -445,7 +445,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } private static class MyIndex extends MapReduceIndex { - public MyIndex(final IndexStorage storage) { + public MyIndex(final IndexStorage storage) throws IOException { super(null, null, storage); } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 0b565e657cd7..58d562543b1c 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -227,7 +227,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi @NotNull @Override public UpdatableIndex createIndexImplementation(@NotNull final ID indexId, @NotNull final FileBasedIndex owner, @NotNull IndexStorage storage) - throws StorageException { + throws StorageException, IOException { if (storage instanceof MemoryIndexStorage) { final MemoryIndexStorage memStorage = (MemoryIndexStorage)storage; memStorage.addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() { @@ -273,7 +273,8 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi private class MyIndex extends MapReduceIndex { private StubIndexImpl myStubIndex; - public MyIndex(final ID indexId, final IndexStorage storage, final DataIndexer indexer) throws StorageException { + public MyIndex(final ID indexId, final IndexStorage storage, final DataIndexer indexer) + throws StorageException, IOException { super(indexId, indexer, storage); checkNameStorage(); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ByteSequenceDataExternalizer.java b/platform/lang-impl/src/com/intellij/util/indexing/ByteSequenceDataExternalizer.java new file mode 100644 index 000000000000..216ad325bee3 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/ByteSequenceDataExternalizer.java @@ -0,0 +1,43 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.util.io.ByteSequence; +import com.intellij.util.io.DataExternalizer; +import org.jetbrains.annotations.NotNull; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.InputStream; + +/** +* Created by Maxim.Mossienko on 5/7/2014. +*/ +class ByteSequenceDataExternalizer implements DataExternalizer { + static final ByteSequenceDataExternalizer INSTANCE = new ByteSequenceDataExternalizer(); + @Override + public void save(@NotNull DataOutput out, ByteSequence value) throws IOException { + out.write(value.getBytes(), value.getOffset(), value.getLength()); // todo fix double copying + } + + @Override + public ByteSequence read(@NotNull DataInput in) throws IOException { + byte[] buf = new byte[((InputStream)in).available()]; // todo fix double copying + in.readFully(buf); + return new ByteSequence(buf); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java b/platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java index 6607745d5b44..40e5d26e702e 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/ContentHashesSupport.java @@ -15,9 +15,7 @@ */ package com.intellij.util.indexing; -import com.intellij.openapi.application.PathManager; import com.intellij.openapi.fileTypes.FileType; -import com.intellij.openapi.util.ThrowableComputable; import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon; import com.intellij.util.io.IOUtil; @@ -33,28 +31,27 @@ import java.security.MessageDigest; * @since 4/10/2014. */ class ContentHashesSupport { - private static final ContentHashesUtil.HashEnumerator ourHashesWithFileType; + private static volatile ContentHashesUtil.HashEnumerator ourHashesWithFileType; - static { - ContentHashesUtil.HashEnumerator hashEnumerator = null; - try { - final File hashEnumeratorFile = new File(PathManager.getIndexRoot(), "hashesWithFileType"); - hashEnumerator = IOUtil.openCleanOrResetBroken(new ThrowableComputable() { - @Override - public ContentHashesUtil.HashEnumerator compute() throws IOException { - return new ContentHashesUtil.HashEnumerator(hashEnumeratorFile, null); - } - }, hashEnumeratorFile); - FlushingDaemon.everyFiveSeconds(new Runnable() { - @Override - public void run() { - if (ourHashesWithFileType.isDirty()) ourHashesWithFileType.force(); - } - }); - } catch (IOException ex) { - throw new RuntimeException(ex); - } finally { - ourHashesWithFileType = hashEnumerator; + static void initContentHashesEnumerator() throws IOException { + if (ourHashesWithFileType != null) return; + synchronized (ContentHashesSupport.class) { + if (ourHashesWithFileType != null) return; + ContentHashesUtil.HashEnumerator hashEnumerator = null; + final File hashEnumeratorFile = new File(IndexInfrastructure.getPersistentIndexRoot(), "hashesWithFileType"); + try { + hashEnumerator = new ContentHashesUtil.HashEnumerator(hashEnumeratorFile, null); + FlushingDaemon.everyFiveSeconds(new Runnable() { + @Override + public void run() { + if (ourHashesWithFileType.isDirty()) ourHashesWithFileType.force(); + } + }); + ourHashesWithFileType = hashEnumerator; + } catch (IOException ex) { + IOUtil.deleteAllFilesStartingWith(hashEnumeratorFile); + throw ex; + } } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java index d504daf8e808..e0e5d8e79db8 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java @@ -21,8 +21,10 @@ package com.intellij.util.indexing; import org.jetbrains.annotations.NotNull; +import java.io.IOException; + public abstract class CustomImplementationFileBasedIndexExtension extends FileBasedIndexExtension { @NotNull public abstract UpdatableIndex createIndexImplementation(@NotNull ID indexId, @NotNull FileBasedIndex owner, @NotNull IndexStorage storage) - throws StorageException; + throws StorageException, IOException; } \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index f9593ebcf58a..41f6fcb83409 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -381,7 +381,10 @@ public class FileBasedIndexImpl extends FileBasedIndex { versionChanged = true; LOG.info("Version has changed for index " + name + ". The index will be rebuilt."); } - FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); + if (extension.hasSnapshotMapping() && (isCurrentVersionCorrupted || versionChanged)) { + safeDelete(IndexInfrastructure.getPersistentIndexRootDir(name)); + } + safeDelete(IndexInfrastructure.getIndexRootDir(name)); IndexingStamp.rewriteVersion(versionFile, version); } @@ -394,8 +397,14 @@ public class FileBasedIndexImpl extends FileBasedIndex { throws IOException { MapIndexStorage storage = null; final ID name = extension.getName(); + boolean contentHashesEnumeratorOk = false; + for (int attempt = 0; attempt < 2; attempt++) { try { + if (extension.hasSnapshotMapping()) { + ContentHashesSupport.initContentHashesEnumerator(); + contentHashesEnumeratorOk = true; + } storage = new MapIndexStorage( IndexInfrastructure.getStorageFile(name), extension.getKeyDescriptor(), @@ -443,6 +452,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { } catch (Exception e) { LOG.info(e); + boolean instantiatedStorage = storage != null; try { if (storage != null) storage.close(); storage = null; @@ -450,12 +460,22 @@ public class FileBasedIndexImpl extends FileBasedIndex { catch (Exception ignored) { } - FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); + safeDelete(IndexInfrastructure.getIndexRootDir(name)); + + if (extension.hasSnapshotMapping() && (!contentHashesEnumeratorOk || instantiatedStorage)) { + safeDelete(IndexInfrastructure.getPersistentIndexRootDir(name)); // todo there is possibility of corruption of storage and content hashes + } IndexingStamp.rewriteVersion(versionFile, version); } } } + private static boolean safeDelete(File dir) { + File directory = FileUtil.findSequentNonexistentFile(dir.getParentFile(), dir.getName(), ""); + boolean success = dir.renameTo(directory); + return FileUtil.delete(success ? directory:dir); + } + private static void saveRegisteredIndices(@NotNull Collection> ids) { final File file = getRegisteredIndicesFile(); try { @@ -519,7 +539,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { extension.hasSnapshotMapping() && IdIndex.ourSnapshotMappingsEnabled ? createInputsIndexExternalizer(extension, indexId, extension.getKeyDescriptor()) : null; - index = new MapReduceIndex(indexId, extension.getIndexer(), storage, externalizer); + index = new MapReduceIndex(indexId, extension.getIndexer(), storage, externalizer, extension.getValueExternalizer()); } index.setInputIdToDataKeysIndex(new Factory>>() { @Override @@ -1620,7 +1640,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { indicesToDrop.remove(key.toString()); } for (String s : indicesToDrop) { - FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s))); + safeDelete(IndexInfrastructure.getIndexRootDir(ID.create(s))); } } @@ -1715,7 +1735,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { byte[] hash; try { currentBytes = content.getBytes(); - hash = fileType.isBinary() ? null:ContentHashesSupport.calcContentHashWithFileType(currentBytes, fileType); + hash = fileType.isBinary() || !IdIndex.ourSnapshotMappingsEnabled ? null:ContentHashesSupport.calcContentHashWithFileType(currentBytes, fileType); } catch (IOException e) { currentBytes = ArrayUtil.EMPTY_BYTE_ARRAY; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java index f795c3594dba..7a70278e3c33 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java @@ -36,6 +36,7 @@ import java.util.Locale; public class IndexInfrastructure { private static final boolean ourUnitTestMode = ApplicationManager.getApplication().isUnitTestMode(); private static final String STUB_VERSIONS = ".versions"; + private static final String PERSISTENT_INDEX_DIRECTORY_NAME = ".persistent"; private IndexInfrastructure() { } @@ -60,14 +61,35 @@ public class IndexInfrastructure { return getIndexDirectory(indexName, false); } + public static File getPersistentIndexRoot() { + File indexDir = new File(PathManager.getIndexRoot() + File.separator + PERSISTENT_INDEX_DIRECTORY_NAME); + indexDir.mkdirs(); + return indexDir; + } + + @NotNull + public static File getPersistentIndexRootDir(@NotNull ID indexName) { + return getIndexDirectory(indexName, false, PERSISTENT_INDEX_DIRECTORY_NAME); + } + @NotNull private static File getIndexDirectory(@NotNull ID indexName, boolean forVersion) { + return getIndexDirectory(indexName, forVersion, ""); + } + + @NotNull + private static File getIndexDirectory(@NotNull ID indexName, boolean forVersion, String relativePath) { final String dirName = indexName.toString().toLowerCase(Locale.US); - // store StubIndices under StubUpdating index' root to ensure they are deleted - // when StubUpdatingIndex version is changed - final File indexDir = indexName instanceof StubIndexKey - ? new File(getIndexRootDir(StubUpdatingIndex.INDEX_ID), forVersion ? STUB_VERSIONS : dirName) - : new File(PathManager.getIndexRoot(), dirName); + File indexDir; + + if (indexName instanceof StubIndexKey) { + // store StubIndices under StubUpdating index' root to ensure they are deleted + // when StubUpdatingIndex version is changed + indexDir = new File(getIndexDirectory(StubUpdatingIndex.INDEX_ID, false, relativePath), forVersion ? STUB_VERSIONS : dirName); + } else { + if (relativePath.length() > 0) relativePath = File.separator + relativePath; + indexDir = new File(PathManager.getIndexRoot() + relativePath, dirName); + } indexDir.mkdirs(); return indexDir; } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java index 3c186c925448..50bd4de6b96d 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java @@ -19,21 +19,22 @@ package com.intellij.util.indexing; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.util.*; +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; +import com.intellij.openapi.util.io.ByteSequence; import com.intellij.openapi.util.io.FileUtil; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; +import com.intellij.util.SmartList; +import com.intellij.util.SystemProperties; import com.intellij.util.io.*; +import com.intellij.util.io.DataOutputStream; import gnu.trove.THashMap; import gnu.trove.TObjectObjectProcedure; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.Set; +import java.io.*; +import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -48,10 +49,12 @@ public class MapReduceIndex implements UpdatableIndex myIndexer; @NotNull protected final IndexStorage myStorage; private final boolean myHasSnapshotMapping; + + private final DataExternalizer myValueExternalizer; private final DataExternalizer> mySnapshotIndexExternalizer; private PersistentHashMap> myInputsIndex; - private PersistentHashMap> mySnapshotMapping; + private final PersistentHashMap myContents; private PersistentHashMap myInputsSnapshotMapping; private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); @@ -60,19 +63,34 @@ public class MapReduceIndex implements UpdatableIndex indexId, DataIndexer indexer, - @NotNull IndexStorage storage) { - this(indexId, indexer, storage, null); + @NotNull IndexStorage storage) throws IOException { + this(indexId, indexer, storage, null, null); } public MapReduceIndex(@Nullable final ID indexId, DataIndexer indexer, @NotNull IndexStorage storage, - DataExternalizer> snapshotIndexExternalizer) { + DataExternalizer> snapshotIndexExternalizer, + DataExternalizer valueDataExternalizer) throws IOException { myIndexId = indexId; myIndexer = indexer; myStorage = storage; myHasSnapshotMapping = snapshotIndexExternalizer != null; + mySnapshotIndexExternalizer = snapshotIndexExternalizer; + myValueExternalizer = valueDataExternalizer; + final File saved = myHasSnapshotMapping && myIndexId != null ? new File(IndexInfrastructure.getPersistentIndexRootDir(myIndexId), "values") : null; + + if (saved != null) { + try { + myContents = new PersistentHashMap(saved, EnumeratorIntegerDescriptor.INSTANCE, ByteSequenceDataExternalizer.INSTANCE); + } catch (IOException ex) { + IOUtil.deleteAllFilesStartingWith(saved); + throw ex; + } + } else { + myContents = null; + } } @NotNull @@ -93,10 +111,6 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex createInputSnapshotMapping() throws IOException { assert myIndexId != null; final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId"); - return IOUtil.openCleanOrResetBroken(new ThrowableComputable, IOException>() { - @Override - public PersistentHashMap compute() throws IOException { - return new PersistentHashMap(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE, 4096) { - @Override - protected boolean wantCompactIntegralValues() { - return true; - } - }; - } - }, fileIdToHashIdFile); + try { + return new PersistentHashMap(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE, + EnumeratorIntegerDescriptor.INSTANCE, 4096) { + @Override + protected boolean wantCompactIntegralValues() { + return true; + } + }; + } + catch (IOException ex) { + IOUtil.deleteAllFilesStartingWith(fileIdToHashIdFile); + throw ex; + } } private static void cleanMapping(@NotNull PersistentHashMap index) { @@ -142,7 +158,7 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex implements UpdatableIndex> createSnapshotMappingIndex() throws IOException { - assert myIndexId != null; - final File hashIdToKeysFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "hashIdToKeys"); - return IOUtil.openCleanOrResetBroken(new ThrowableComputable>, IOException>() { - @Override - public PersistentHashMap> compute() throws IOException { - return new PersistentHashMap>(hashIdToKeysFile, EnumeratorIntegerDescriptor.INSTANCE, mySnapshotIndexExternalizer); - } - }, hashIdToKeysFile); - } - @Nullable private PersistentHashMap> createInputsIndex() throws IOException { Factory>> factory = myInputsIndexFactory; @@ -275,40 +279,64 @@ public class MapReduceIndex implements UpdatableIndex update(final int inputId, @Nullable Input content) { + final boolean weProcessPhysicalContent = content == null || + (content instanceof FileContent && + ((FileContent)content).getUserData(FileBasedIndexImpl.ourPhysicalContentKey) != null); - final Map data = content != null ? myIndexer.map(content) : Collections.emptyMap(); + Map data = null; + boolean havePersistentData = false; + Integer hashId = null; + if (myContents != null && weProcessPhysicalContent && content != null) { + try { + hashId = getHashOfContent((FileContent)content); + if (doReadSavedPersistentData) { + ByteSequence bytes = myContents.get(hashId); + if (bytes != null) { + data = deserializeSavedPersistentData(bytes); + havePersistentData = true; + } + } else { + havePersistentData = myContents.containsMapping(hashId); + } + } catch (IOException ex) { + // todo: + throw new RuntimeException(ex); + } + } + + if (data == null) data = content != null ? myIndexer.map(content) : Collections.emptyMap(); + + if (hashId != null && !havePersistentData && data.size() > 0) { + savePersistentData(data, hashId); + } ProgressManager.checkCanceled(); final NotNullComputable> oldKeysGetter; final int savedInputId; - - boolean weProcessPhysicalContent = content == null || - (content instanceof FileContent && - ((FileContent)content).getUserData(FileBasedIndexImpl.ourPhysicalContentKey) != null); - if (myHasSnapshotMapping && weProcessPhysicalContent) { - try { // optimistically (out of index update write section) read current snapshot keys for file - final Integer hashId = myInputsSnapshotMapping.get(inputId); - final Collection keys = hashId != null ? mySnapshotMapping.get(hashId): null; + try { oldKeysGetter = new NotNullComputable>() { - @NotNull - @Override - public Collection compute() { + @NotNull + @Override + public Collection compute() { try { Integer currentHashId = myInputsSnapshotMapping.get(inputId); Collection currentKeys; - if ((currentHashId == null && hashId == null) || (currentHashId != null && currentHashId.equals(hashId))) { - currentKeys = keys; - } else { // optimistic reading file's current keys failed - currentKeys = currentHashId != null ? mySnapshotMapping.get(currentHashId): null; + if (currentHashId != null) { + ByteSequence byteSequence = myContents.get(currentHashId); + currentKeys = byteSequence != null ? deserializeSavedPersistentData(byteSequence).keySet() : Collections.emptyList(); + } + else { + currentKeys = Collections.emptyList(); } - return currentKeys == null ? Collections.emptyList() : currentKeys; + return currentKeys; } catch (IOException e) { throw new RuntimeException(e); } @@ -316,22 +344,7 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex finalData = data; return new Computable() { @Override public Boolean compute() { @@ -364,7 +378,7 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex deserializeSavedPersistentData(ByteSequence bytes) throws IOException { + DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes.getBytes(), bytes.getOffset(), bytes.getLength())); + int pairs = DataInputOutputUtil.readINT(stream); + if (pairs == 0) return Collections.emptyMap(); + Map result = new THashMap(pairs); + while (stream.available() > 0) { + Value value = myValueExternalizer.read(stream); + Collection keys = mySnapshotIndexExternalizer.read(stream); + for(Key k:keys) result.put(k, value); + } + return result; + } + + private static Integer getHashOfContent(FileContent content) throws IOException { + Integer previouslyCalculatedContentHashId = content.getUserData(ourSavedContentHashIdKey); + if (previouslyCalculatedContentHashId == null) { + byte[] hash = content instanceof FileContentImpl ? ((FileContentImpl)content).getHash():null; + if (hash == null) { + previouslyCalculatedContentHashId = ContentHashesSupport + .calcContentHashIdWithFileType(content.getContent(), content.getFileType()); + } else { + previouslyCalculatedContentHashId = ContentHashesSupport.enumerateHash(hash); + } + content.putUserData(ourSavedContentHashIdKey, previouslyCalculatedContentHashId); + } + return previouslyCalculatedContentHashId; + } + + private static final ThreadLocalCachedByteArray ourSpareByteArray = new ThreadLocalCachedByteArray(); + + private void savePersistentData(Map data, int id) { + try { + BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(ourSpareByteArray.getBuffer(4 * data.size())); + DataOutputStream stream = new DataOutputStream(out); + int size = data.size(); + DataInputOutputUtil.writeINT(stream, size); + + if (size > 0) { + THashMap> values = new THashMap>(); + List keysForNullValue = null; + for (Map.Entry e : data.entrySet()) { + Value value = e.getValue(); + + List keys = value != null ? values.get(value):keysForNullValue; + if (keys == null) { + if (value != null) values.put(value, keys = new SmartList()); + else keys = keysForNullValue = new SmartList(); + } + keys.add(e.getKey()); + } + + if (keysForNullValue != null) { + myValueExternalizer.save(stream, null); + mySnapshotIndexExternalizer.save(stream, keysForNullValue); + } + + for(Value value:values.keySet()) { + myValueExternalizer.save(stream, value); + mySnapshotIndexExternalizer.save(stream, values.get(value)); + } + } + + myContents.put(id, new ByteSequence(out.getInternalBuffer(), 0, out.size())); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + private static final com.intellij.openapi.util.Key ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id"); protected void updateWithMap(final int inputId, diff --git a/platform/platform-api/src/com/intellij/util/net/HttpConfigurable.java b/platform/platform-api/src/com/intellij/util/net/HttpConfigurable.java index 4dfec8669bf9..94eaa4db1ccb 100644 --- a/platform/platform-api/src/com/intellij/util/net/HttpConfigurable.java +++ b/platform/platform-api/src/com/intellij/util/net/HttpConfigurable.java @@ -62,7 +62,7 @@ import java.util.*; }, storageChooser = HttpConfigurable.StorageChooser.class ) -public class HttpConfigurable implements PersistentStateComponent, ApplicationComponent, JDOMExternalizable, +public class HttpConfigurable implements PersistentStateComponent, ApplicationComponent, ExportableApplicationComponent { private static final Logger LOG = Logger.getInstance("#com.intellij.util.net.HttpConfigurable"); public boolean PROXY_TYPE_IS_SOCKS = false; @@ -98,6 +98,7 @@ public class HttpConfigurable implements PersistentStateComponentc = new StringBuilder(); c.append(' '); } @@ -17,4 +17,17 @@ public class MismatchedStringBuilderQueryUpdate { sb.append("abc"); return sb.reverse(); } + + void indexedList(List list) { + StringBuilder stringBuilder = new StringBuilder(); // <--- false warning here + list.forEach(stringBuilder::append); + System.out.println(stringBuilder.toString()); + } +} +interface List { + default void forEach(Consumer action) { + } +} +interface Consumer { + void accept(T t); } diff --git a/plugins/InspectionGadgets/test/com/siyeh/igtest/bugs/mismatched_string_builder_query_update/expected.xml b/plugins/InspectionGadgets/test/com/siyeh/igtest/bugs/mismatched_string_builder_query_update/expected.xml deleted file mode 100644 index 43998eaeb786..000000000000 --- a/plugins/InspectionGadgets/test/com/siyeh/igtest/bugs/mismatched_string_builder_query_update/expected.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - MismatchedStringBuilderQueryUpdate.java - 10 - Mismatched query and update of StringBuilder - Contents of StringBuilder <code>c</code> are updated, but never queried #loc - - \ No newline at end of file diff --git a/plugins/InspectionGadgets/testsrc/com/siyeh/ig/bugs/MismatchedStringBuilderQueryUpdateInspectionTest.java b/plugins/InspectionGadgets/testsrc/com/siyeh/ig/bugs/MismatchedStringBuilderQueryUpdateInspectionTest.java index 4a739a04234a..293d372d6ca6 100644 --- a/plugins/InspectionGadgets/testsrc/com/siyeh/ig/bugs/MismatchedStringBuilderQueryUpdateInspectionTest.java +++ b/plugins/InspectionGadgets/testsrc/com/siyeh/ig/bugs/MismatchedStringBuilderQueryUpdateInspectionTest.java @@ -1,10 +1,16 @@ package com.siyeh.ig.bugs; -import com.siyeh.ig.IGInspectionTestCase; +import com.intellij.codeInspection.InspectionProfileEntry; +import com.siyeh.ig.LightInspectionTestCase; -public class MismatchedStringBuilderQueryUpdateInspectionTest extends IGInspectionTestCase { +public class MismatchedStringBuilderQueryUpdateInspectionTest extends LightInspectionTestCase { - public void test() throws Exception { - doTest("com/siyeh/igtest/bugs/mismatched_string_builder_query_update", new MismatchedStringBuilderQueryUpdateInspection()); + public void testMismatchedStringBuilderQueryUpdate() throws Exception { + doTest(); + } + + @Override + protected InspectionProfileEntry getInspection() { + return new MismatchedStringBuilderQueryUpdateInspection(); } } \ No newline at end of file diff --git a/plugins/devkit/src/inspections/DevKitInspectionBase.java b/plugins/devkit/src/inspections/DevKitInspectionBase.java index 9ded8e216fc0..5b58a7af8287 100644 --- a/plugins/devkit/src/inspections/DevKitInspectionBase.java +++ b/plugins/devkit/src/inspections/DevKitInspectionBase.java @@ -26,9 +26,12 @@ import com.intellij.psi.*; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.xml.*; import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.xml.DomFileElement; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.idea.devkit.DevKitBundle; +import org.jetbrains.idea.devkit.dom.Dependency; +import org.jetbrains.idea.devkit.dom.IdeaPlugin; import org.jetbrains.idea.devkit.module.PluginModuleType; import org.jetbrains.idea.devkit.util.ActionType; import org.jetbrains.idea.devkit.util.ComponentType; @@ -62,7 +65,8 @@ public abstract class DevKitInspectionBase extends BaseJavaLocalInspectionTool { if (PluginModuleType.isOfType(module)) { return checkModule(module, psiClass, null, includeActions); - } else { + } + else { Set types = null; final List modules = PluginModuleType.getCandidateModules(module); for (Module m : modules) { @@ -78,20 +82,26 @@ public abstract class DevKitInspectionBase extends BaseJavaLocalInspectionTool { if (!DescriptorUtil.isPluginXml(pluginXml)) return types; assert pluginXml != null; - final XmlDocument document = pluginXml.getDocument(); - assert document != null; - - final XmlTag rootTag = document.getRootTag(); - assert rootTag != null; - final String qualifiedName = psiClass.getQualifiedName(); if (qualifiedName != null) { final RegistrationTypeFinder finder = new RegistrationTypeFinder(psiClass, types); - DescriptorUtil.processComponents(rootTag, finder); + // "main" plugin.xml + processPluginXml(pluginXml, finder, includeActions); - if (includeActions) { - DescriptorUtil.processActions(rootTag, finder); + // plugin.xml files + final VirtualFile pluginXmlDirectory = pluginXml.getVirtualFile().getParent(); + final DomFileElement fileElement = DescriptorUtil.getIdeaPlugin(pluginXml); + for (Dependency dependency : fileElement.getRootElement().getDependencies()) { + final String configFile = dependency.getConfigFile().getValue(); + if (configFile != null) { + final VirtualFile depVirtualFile = pluginXmlDirectory.findChild(configFile); + if (depVirtualFile == null) continue; + final PsiFile depPluginXml = PsiManager.getInstance(module.getProject()).findFile(depVirtualFile); + if (DescriptorUtil.isPluginXml(depPluginXml)) { + processPluginXml((XmlFile)depPluginXml, finder, includeActions); + } + } } types = finder.getTypes(); @@ -100,6 +110,18 @@ public abstract class DevKitInspectionBase extends BaseJavaLocalInspectionTool { return types; } + private static void processPluginXml(XmlFile xmlFile, RegistrationTypeFinder finder, boolean includeActions) { + final XmlDocument document = xmlFile.getDocument(); + if (document == null) return; + final XmlTag rootTag = document.getRootTag(); + if (rootTag == null) return; + + DescriptorUtil.processComponents(rootTag, finder); + if (includeActions) { + DescriptorUtil.processActions(rootTag, finder); + } + } + @Nullable protected static PsiElement getAttValueToken(@NotNull XmlAttribute attribute) { final XmlAttributeValue valueElement = attribute.getValueElement(); @@ -132,13 +154,14 @@ public abstract class DevKitInspectionBase extends BaseJavaLocalInspectionTool { return false; } - static class RegistrationTypeFinder implements ComponentType.Processor, ActionType.Processor { + + private static class RegistrationTypeFinder implements ComponentType.Processor, ActionType.Processor { private Set myTypes; private final String myQualifiedName; private final PsiManager myManager; private final GlobalSearchScope myScope; - public RegistrationTypeFinder(PsiClass psiClass, Set types) { + private RegistrationTypeFinder(PsiClass psiClass, Set types) { myTypes = types; myQualifiedName = psiClass.getQualifiedName(); myManager = psiClass.getManager(); diff --git a/plugins/devkit/src/util/DescriptorUtil.java b/plugins/devkit/src/util/DescriptorUtil.java index 3a773e9e5fa9..3cf574f9ca48 100644 --- a/plugins/devkit/src/util/DescriptorUtil.java +++ b/plugins/devkit/src/util/DescriptorUtil.java @@ -87,7 +87,7 @@ public class DescriptorUtil { return getIdeaPlugin((XmlFile)file) != null; } - private static DomFileElement getIdeaPlugin(XmlFile file) { + public static DomFileElement getIdeaPlugin(XmlFile file) { return DomManager.getDomManager(file.getProject()).getFileElement(file, IdeaPlugin.class); } } diff --git a/plugins/devkit/testData/inspections/componentNotRegistered/registeredActionInOptionalPluginDescriptor-optional-plugin.xml b/plugins/devkit/testData/inspections/componentNotRegistered/registeredActionInOptionalPluginDescriptor-optional-plugin.xml new file mode 100644 index 000000000000..e2216421f271 --- /dev/null +++ b/plugins/devkit/testData/inspections/componentNotRegistered/registeredActionInOptionalPluginDescriptor-optional-plugin.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/plugins/devkit/testData/inspections/componentNotRegistered/registeredActionInOptionalPluginDescriptor-plugin.xml b/plugins/devkit/testData/inspections/componentNotRegistered/registeredActionInOptionalPluginDescriptor-plugin.xml new file mode 100644 index 000000000000..0ace22f7d835 --- /dev/null +++ b/plugins/devkit/testData/inspections/componentNotRegistered/registeredActionInOptionalPluginDescriptor-plugin.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/plugins/devkit/testSources/inspections/ComponentNotRegisteredInspectionTest.java b/plugins/devkit/testSources/inspections/ComponentNotRegisteredInspectionTest.java index 238c6b294877..3b365554d0c2 100644 --- a/plugins/devkit/testSources/inspections/ComponentNotRegisteredInspectionTest.java +++ b/plugins/devkit/testSources/inspections/ComponentNotRegisteredInspectionTest.java @@ -67,6 +67,14 @@ public class ComponentNotRegisteredInspectionTest extends LightCodeInsightFixtur myFixture.testHighlighting("RegisteredAction.java"); } + public void testRegisteredActionInOptionalPluginDescriptor() { + setPluginXml("registeredActionInOptionalPluginDescriptor-plugin.xml"); + myFixture.copyFileToProject("registeredActionInOptionalPluginDescriptor-optional-plugin.xml", + "META-INF/optional-plugin.xml"); + + myFixture.testHighlighting("RegisteredAction.java"); + } + public void testUnregisteredAction() { setPluginXml("unregisteredAction-plugin.xml"); myFixture.testHighlighting("UnregisteredAction.java");