From feb9b635d8bc0b99e82e0806ae38f6077da8c535 Mon Sep 17 00:00:00 2001 From: Alexey Kudravtsev Date: Mon, 7 Apr 2014 13:55:10 +0400 Subject: [PATCH] IDEA-71597 --- .../com/intellij/find/FindManagerTest.java | 6 +- .../testSrc/com/intellij/index/IndexTest.java | 14 +- .../openapi/fileTypes/FileTypeRegistry.java | 4 +- .../openapi/fileEditor/impl/LoadTextUtil.java | 22 +- .../psi/SingleRootFileViewProvider.java | 4 - .../intellij/find/impl/FindInProjectTask.java | 6 +- .../util/indexing/FileBasedIndexImpl.java | 159 +++++++------ .../intellij/util/indexing/IndexingStamp.java | 2 +- .../intellij/openapi/diff/FileContent.java | 30 +-- .../openapi/vfs/newvfs/FileAttribute.java | 13 +- .../intellij/ide/dnd/FileCopyPasteUtil.java | 6 +- .../impl/FileDocumentManagerImpl.java | 31 +-- .../fileTypes/impl/FileTypeManagerImpl.java | 216 ++++++++++++------ .../fileTypes/FileTypeManagerTest.java | 18 +- .../openapi/fileTypes/FileTypesTest.java | 30 ++- .../util/containers/ConcurrentBitSet.java | 74 +++++- .../intellij/openapi/vcs/FilePathImpl.java | 13 +- 17 files changed, 381 insertions(+), 267 deletions(-) diff --git a/java/java-tests/testSrc/com/intellij/find/FindManagerTest.java b/java/java-tests/testSrc/com/intellij/find/FindManagerTest.java index 4794d1a5c6ef..86d79fb41e10 100644 --- a/java/java-tests/testSrc/com/intellij/find/FindManagerTest.java +++ b/java/java-tests/testSrc/com/intellij/find/FindManagerTest.java @@ -28,7 +28,7 @@ import com.intellij.openapi.command.CommandProcessor; import com.intellij.openapi.command.WriteCommandAction; import com.intellij.openapi.fileEditor.FileDocumentManager; import com.intellij.openapi.fileTypes.FileTypes; -import com.intellij.openapi.fileTypes.impl.FileTypeManagerImpl; +import com.intellij.openapi.fileTypes.PlainTextFileType; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.util.ProperTextRange; import com.intellij.openapi.util.io.FileUtil; @@ -302,8 +302,7 @@ public class FindManagerTest extends DaemonAnalyzerTestCase { }.execute().getResultObject(); assertNull(FileDocumentManager.getInstance().getCachedDocument(custom)); - assertEquals(FileTypes.UNKNOWN, custom.getFileType()); - assertFalse(FileTypeManagerImpl.isFileTypeDetectedFromContent(custom)); + assertEquals(PlainTextFileType.INSTANCE, custom.getFileType()); FindModel findModel = new FindModel(); findModel.setWholeWordsOnly(true); @@ -318,7 +317,6 @@ public class FindManagerTest extends DaemonAnalyzerTestCase { // and we should get the same with text loaded assertNotNull(FileDocumentManager.getInstance().getDocument(custom)); assertEquals(FileTypes.PLAIN_TEXT, custom.getFileType()); - assertTrue(FileTypeManagerImpl.isFileTypeDetectedFromContent(custom)); assertSize(2, findUsages(findModel)); } diff --git a/java/java-tests/testSrc/com/intellij/index/IndexTest.java b/java/java-tests/testSrc/com/intellij/index/IndexTest.java index 7770c5e4a1f9..4fcb40296a48 100644 --- a/java/java-tests/testSrc/com/intellij/index/IndexTest.java +++ b/java/java-tests/testSrc/com/intellij/index/IndexTest.java @@ -18,7 +18,7 @@ package com.intellij.index; import com.intellij.openapi.command.WriteCommandAction; import com.intellij.openapi.editor.Document; import com.intellij.openapi.fileEditor.FileDocumentManager; -import com.intellij.openapi.fileTypes.UnknownFileType; +import com.intellij.openapi.fileTypes.PlainTextFileType; import com.intellij.openapi.util.Factory; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.vfs.VfsUtil; @@ -276,8 +276,8 @@ public class IndexTest extends IdeaTestCase { final VirtualFile vFile = createChildData(dir, "Foo.test"); VfsUtil.saveText(vFile, "Foo"); - assertEquals(UnknownFileType.INSTANCE, vFile.getFileType()); - assertEmpty(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); + assertEquals(PlainTextFileType.INSTANCE, vFile.getFileType()); + assertOneElement(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); final Document document = FileDocumentManager.getInstance().getDocument(vFile); //todo should file type be changed silently without events? @@ -287,22 +287,22 @@ public class IndexTest extends IdeaTestCase { assertInstanceOf(file, PsiPlainTextFile.class); assertEquals("Foo", file.getText()); - assertEmpty(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); + assertOneElement(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); WriteCommandAction.runWriteCommandAction(myProject, new Runnable() { @Override public void run() { document.insertString(0, " "); assertEquals("Foo", file.getText()); - assertEmpty(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); + assertOneElement(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); FileDocumentManager.getInstance().saveDocument(document); assertEquals("Foo", file.getText()); - assertEmpty(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); + assertOneElement(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); PsiDocumentManager.getInstance(myProject).commitAllDocuments(); assertEquals(" Foo", file.getText()); - assertEmpty(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); + assertOneElement(PsiSearchHelper.SERVICE.getInstance(myProject).findFilesWithPlainTextWords("Foo")); } }); diff --git a/platform/core-api/src/com/intellij/openapi/fileTypes/FileTypeRegistry.java b/platform/core-api/src/com/intellij/openapi/fileTypes/FileTypeRegistry.java index 38f26df67e1e..e1a52796e1e9 100644 --- a/platform/core-api/src/com/intellij/openapi/fileTypes/FileTypeRegistry.java +++ b/platform/core-api/src/com/intellij/openapi/fileTypes/FileTypeRegistry.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2011 JetBrains s.r.o. + * Copyright 2000-2014 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -76,8 +76,10 @@ public abstract class FileTypeRegistry { * @return {@link com.intellij.openapi.fileTypes.PlainTextFileType} if file looks like text, * or another file type if some file type detector identified the file * or the {@link UnknownFileType} if file is binary or we are unable to detect. + * @deprecated use {@link VirtualFile#getFileType()} instead */ @NotNull + @Deprecated public abstract FileType detectFileTypeFromContent(@NotNull VirtualFile file); /** diff --git a/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java b/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java index a0baadbb8096..caf5811682aa 100644 --- a/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java +++ b/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java @@ -109,6 +109,10 @@ public final class LoadTextUtil { } private static Charset detectCharset(@NotNull VirtualFile virtualFile, @NotNull byte[] content) { + return detectCharset(virtualFile, content, virtualFile.getFileType()); + } + + public static Charset detectCharset(@NotNull VirtualFile virtualFile, @NotNull byte[] content, @NotNull FileType fileType) { Charset charset = null; Trinity guessed = guessFromContent(virtualFile, content, content.length); @@ -116,7 +120,6 @@ public final class LoadTextUtil { charset = guessed.first; } else { - FileType fileType = virtualFile.getFileType(); String charsetName = fileType.getCharset(virtualFile, content); if (charsetName == null) { @@ -131,7 +134,7 @@ public final class LoadTextUtil { } charset = charset == null ? EncodingRegistry.getInstance().getDefaultCharset() : charset; - if (EncodingRegistry.getInstance().isNative2Ascii(virtualFile)) { + if (fileType.getName().equals("Properties") && EncodingRegistry.getInstance().isNative2AsciiForPropertiesFiles()) { charset = Native2AsciiCharset.wrap(charset); } virtualFile.setCharset(charset); @@ -145,7 +148,11 @@ public final class LoadTextUtil { @NotNull private static Pair doDetectCharsetAndSetBOM(@NotNull VirtualFile virtualFile, @NotNull byte[] content, boolean saveBOM) { - Charset charset = virtualFile.isCharsetSet() ? virtualFile.getCharset() : detectCharset(virtualFile, content); + return doDetectCharsetAndSetBOM(virtualFile, content, saveBOM, virtualFile.getFileType()); + } + @NotNull + private static Pair doDetectCharsetAndSetBOM(@NotNull VirtualFile virtualFile, @NotNull byte[] content, boolean saveBOM, @NotNull FileType fileType) { + Charset charset = virtualFile.isCharsetSet() ? virtualFile.getCharset() : detectCharset(virtualFile, content,fileType); Pair bomAndCharset = getBOMAndCharset(content, charset); final byte[] bom = bomAndCharset.second; if (saveBOM && bom != null && bom.length != 0) { @@ -383,7 +390,14 @@ public final class LoadTextUtil { @NotNull VirtualFile virtualFile, boolean saveDetectedSeparators, boolean saveBOM) { - Pair pair = doDetectCharsetAndSetBOM(virtualFile, bytes, saveBOM); + return getTextByBinaryPresentation(bytes, virtualFile, saveDetectedSeparators, saveBOM, virtualFile.getFileType()); + } + @NotNull + public static CharSequence getTextByBinaryPresentation(@NotNull byte[] bytes, + @NotNull VirtualFile virtualFile, + boolean saveDetectedSeparators, + boolean saveBOM, @NotNull FileType fileType) { + Pair pair = doDetectCharsetAndSetBOM(virtualFile, bytes, saveBOM, fileType); Charset charset = pair.getFirst(); byte[] bom = pair.getSecond(); int offset = bom == null ? 0 : bom.length; diff --git a/platform/core-impl/src/com/intellij/psi/SingleRootFileViewProvider.java b/platform/core-impl/src/com/intellij/psi/SingleRootFileViewProvider.java index 8c59b0999d0e..f096dc3c0e88 100644 --- a/platform/core-impl/src/com/intellij/psi/SingleRootFileViewProvider.java +++ b/platform/core-impl/src/com/intellij/psi/SingleRootFileViewProvider.java @@ -105,10 +105,6 @@ public class SingleRootFileViewProvider extends UserDataHolderBase implements Fi } FileType fileType = file.getFileType(); - // Do not load content - if (fileType == UnknownFileType.INSTANCE) { - fileType = FileTypeRegistry.getInstance().detectFileTypeFromContent(file); - } if (fileType.isBinary()) return Language.ANY; if (isTooLargeForIntelligence(file)) return PlainTextLanguage.INSTANCE; diff --git a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java index 0f50eed31d3f..bbd24f9d864e 100644 --- a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java +++ b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java @@ -23,6 +23,7 @@ import com.intellij.find.ngrams.TrigramIndex; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.ApplicationNamesInfo; import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.fileTypes.FileTypeManager; import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleManager; @@ -298,8 +299,9 @@ class FindInProjectTask { } private static boolean isCoveredByIdIndex(VirtualFile file) { - return IdIndex.isIndexable(FileBasedIndexImpl.getFileType(file)) && - ((FileBasedIndexImpl)FileBasedIndex.getInstance()).isIndexingCandidate(file, IdIndex.NAME); + FileBasedIndexImpl fileBasedIndex = (FileBasedIndexImpl)FileBasedIndex.getInstance(); + FileType fileType = file.getFileType(); + return IdIndex.isIndexable(fileType) && fileBasedIndex.isIndexingCandidate(file, IdIndex.NAME); } private static boolean iterateAll(@NotNull VirtualFile[] files, @NotNull final GlobalSearchScope searchScope, @NotNull final ContentIterator iterator) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 64f67521409f..9d7a690cf27b 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -66,7 +66,6 @@ import com.intellij.psi.impl.cache.impl.id.PlatformIdTableBuilding; import com.intellij.psi.impl.source.PsiFileImpl; import com.intellij.psi.search.EverythingGlobalScope; import com.intellij.psi.search.GlobalSearchScope; -import com.intellij.psi.stubs.SerializationManager; import com.intellij.psi.stubs.SerializationManagerEx; import com.intellij.util.*; import com.intellij.util.concurrency.Semaphore; @@ -128,7 +127,8 @@ public class FileBasedIndexImpl extends FileBasedIndex { private final MessageBusConnection myConnection; private final FileDocumentManager myFileDocumentManager; - private final FileTypeManager myFileTypeManager; + private final FileTypeManagerImpl myFileTypeManager; + private final SerializationManagerEx mySerializationManagerEx; private final ConcurrentHashSet> myUpToDateIndicesForUnsavedOrTransactedDocuments = new ConcurrentHashSet>(); private volatile SmartFMap myTransactionMap = SmartFMap.emptyMap(); @@ -146,11 +146,12 @@ public class FileBasedIndexImpl extends FileBasedIndex { public FileBasedIndexImpl(@SuppressWarnings("UnusedParameters") VirtualFileManager vfManager, FileDocumentManager fdm, - FileTypeManager fileTypeManager, + FileTypeManagerImpl fileTypeManager, @NotNull MessageBus bus, - @SuppressWarnings("UnusedParameters") SerializationManager sm /*needed to ensure dependency*/) { + SerializationManagerEx sm) { myFileDocumentManager = fdm; myFileTypeManager = fileTypeManager; + mySerializationManagerEx = sm; myIsUnitTestMode = ApplicationManager.getApplication().isUnitTestMode(); myConfigPath = calcConfigPath(PathManager.getConfigPath()); myLogPath = calcConfigPath(PathManager.getLogPath()); @@ -175,7 +176,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { @Nullable private Map> myTypeToExtensionMap; @Override - public void beforeFileTypesChanged(final FileTypeEvent event) { + public void beforeFileTypesChanged(@NotNull final FileTypeEvent event) { cleanupProcessedFlag(); myTypeToExtensionMap = new THashMap>(); for (FileType type : myFileTypeManager.getRegisteredFileTypes()) { @@ -184,7 +185,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { } @Override - public void fileTypesChanged(final FileTypeEvent event) { + public void fileTypesChanged(@NotNull final FileTypeEvent event) { final Map> oldExtensions = myTypeToExtensionMap; myTypeToExtensionMap = null; if (oldExtensions != null) { @@ -255,8 +256,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { myConnection = connection; } - public static boolean isProjectOrWorkspaceFile(@NotNull VirtualFile file, - @Nullable FileType fileType) { + public static boolean isProjectOrWorkspaceFile(@NotNull VirtualFile file, @Nullable FileType fileType) { if (fileType instanceof InternalFileType) return true; VirtualFile parent = file.isDirectory() ? file: file.getParent(); while(parent instanceof VirtualFileSystemEntry) { @@ -697,7 +697,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { } if (!HeavyProcessLatch.INSTANCE.isRunning() && modCount == myLocalModCount) { // do not interfere with 'main' jobs - SerializationManagerEx.getInstanceEx().flushNameStorage(); + mySerializationManagerEx.flushNameStorage(); } } @@ -1550,7 +1550,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { final long previousDocStamp = myLastIndexedDocStamps.getAndSet(document, requestedIndexId, currentDocStamp); if (currentDocStamp != previousDocStamp) { final CharSequence contentText = content.getText(); - FileTypeManagerImpl.cacheFileType(vFile, getFileType(vFile)); + FileTypeManagerImpl.cacheFileType(vFile, vFile.getFileType()); try { if (!isTooLarge(vFile, contentText.length()) && getAffectedIndexCandidates(vFile).contains(requestedIndexId) && @@ -1713,7 +1713,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); final VirtualFile file = content.getVirtualFile(); - FileTypeManagerImpl.cacheFileType(file, getFileType(file)); + FileTypeManagerImpl.cacheFileType(file, file.getFileType()); try { PsiFile psiFile = null; @@ -1766,36 +1766,28 @@ public class FileBasedIndexImpl extends FileBasedIndex { } } - @NotNull - public static FileType getFileType(@NotNull VirtualFile file) { - FileType fileType = file.getFileType(); - if (fileType == FileTypes.PLAIN_TEXT && FileTypeManagerImpl.isFileTypeDetectedFromContent(file)) { - fileType = FileTypes.UNKNOWN; - } - return fileType; - } - - public boolean isIndexingCandidate(VirtualFile file, ID indexId) { + public boolean isIndexingCandidate(@NotNull VirtualFile file, @NotNull ID indexId) { return !isTooLarge(file) && getAffectedIndexCandidates(file).contains(indexId); } - private List> getAffectedIndexCandidates(VirtualFile file) { + @NotNull + private List> getAffectedIndexCandidates(@NotNull VirtualFile file) { if (file.isDirectory()) { return isProjectOrWorkspaceFile(file, null) ? Collections.>emptyList() : myIndicesForDirectories; } - FileType fileType = getFileType(file); + FileType fileType = file.getFileType(); if(isProjectOrWorkspaceFile(file, fileType)) return Collections.emptyList(); List> ids = myFileType2IndicesWithFileTypeInfoMap.get(fileType); if (ids == null) ids = myIndicesWithoutFileTypeInfo; return ids; } - private static void cleanFileContent(FileContentImpl fc, PsiFile psiFile) { + private static void cleanFileContent(@NotNull FileContentImpl fc, PsiFile psiFile) { if (psiFile != null) psiFile.putUserData(PsiFileImpl.BUILDING_STUB, false); fc.putUserData(IndexingDataKeys.PSI_FILE, null); } - private static void initFileContent(FileContentImpl fc, Project project, PsiFile psiFile) { + private static void initFileContent(@NotNull FileContentImpl fc, Project project, PsiFile psiFile) { if (psiFile != null) { psiFile.putUserData(PsiFileImpl.BUILDING_STUB, true); fc.putUserData(IndexingDataKeys.PSI_FILE, psiFile); @@ -1862,26 +1854,28 @@ public class FileBasedIndexImpl extends FileBasedIndex { }; } - private void scheduleUpdate(ID indexId, final Computable update, final Runnable successRunnable) { + private void scheduleUpdate(@NotNull ID indexId, @NotNull Computable update, @NotNull Runnable successRunnable) { if (myNotRequiringContentIndices.contains(indexId)) { myContentlessIndicesUpdateQueue.submit(update, successRunnable); - } else { + } + else { Boolean result = update.compute(); if (result == Boolean.TRUE) ApplicationManager.getApplication().runReadAction(successRunnable); } } - private boolean needsFileContentLoading(ID indexId) { + private boolean needsFileContentLoading(@NotNull ID indexId) { return !myNotRequiringContentIndices.contains(indexId); } private abstract static class InvalidationTask implements Runnable { private final VirtualFile mySubj; - protected InvalidationTask(final VirtualFile subj) { + protected InvalidationTask(@NotNull VirtualFile subj) { mySubj = subj; } + @NotNull public VirtualFile getSubj() { return mySubj; } @@ -2013,7 +2007,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { // can be client that used indices between before and after events, in such case indices are up to date due to force update // with old content) if (!fileIsDirectory && !isTooLarge(file)) { - FileTypeManagerImpl.cacheFileType(file, getFileType(file)); + FileTypeManagerImpl.cacheFileType(file, file.getFileType()); try { final List> candidates = getAffectedIndexCandidates(file); //noinspection ForLoopReplaceableByForEach @@ -2374,66 +2368,68 @@ public class FileBasedIndexImpl extends FileBasedIndex { return true; } - if (file instanceof VirtualFileWithId) { - try { - FileTypeManagerImpl.cacheFileType(file, getFileType(file)); + if (!(file instanceof VirtualFileWithId)) { + return true; + } + try { + FileType type = file.getFileType(); + FileTypeManagerImpl.cacheFileType(file, type); - boolean oldStuff = true; - if (file.isDirectory() || !isTooLarge(file)) { - final List> affectedIndexCandidates = getAffectedIndexCandidates(file); - //noinspection ForLoopReplaceableByForEach - for (int i = 0, size = affectedIndexCandidates.size(); i < size; ++i) { - final ID indexId = affectedIndexCandidates.get(i); - try { - if (needsFileContentLoading(indexId) && shouldIndexFile(file, indexId)) { - myFiles.add(file); - oldStuff = false; - break; - } - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException || cause instanceof StorageException) { - LOG.info(e); - requestRebuild(indexId); - } - else { - throw e; - } + boolean oldStuff = true; + if (file.isDirectory() || !isTooLarge(file)) { + final List> affectedIndexCandidates = getAffectedIndexCandidates(file); + //noinspection ForLoopReplaceableByForEach + for (int i = 0, size = affectedIndexCandidates.size(); i < size; ++i) { + final ID indexId = affectedIndexCandidates.get(i); + try { + if (needsFileContentLoading(indexId) && shouldIndexFile(file, indexId)) { + myFiles.add(file); + oldStuff = false; + break; } } - } - FileContent fileContent = null; - for (ID indexId : myNotRequiringContentIndices) { - if (shouldIndexFile(file, indexId)) { - oldStuff = false; - try { - if (fileContent == null) { - fileContent = new FileContentImpl(file); - } - updateSingleIndex(indexId, file, fileContent); - } - catch (StorageException e) { + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof IOException || cause instanceof StorageException) { LOG.info(e); requestRebuild(indexId); } + else { + throw e; + } } } - IndexingStamp.flushCache(file); - - if (oldStuff && file instanceof VirtualFileSystemEntry) { - ((VirtualFileSystemEntry)file).setFileIndexed(true); + } + FileContent fileContent = null; + for (ID indexId : myNotRequiringContentIndices) { + if (shouldIndexFile(file, indexId)) { + oldStuff = false; + try { + if (fileContent == null) { + fileContent = new FileContentImpl(file); + } + updateSingleIndex(indexId, file, fileContent); + } + catch (StorageException e) { + LOG.info(e); + requestRebuild(indexId); + } } } - finally { - FileTypeManagerImpl.cacheFileType(file, null); - } + IndexingStamp.flushCache(file); - if (myProgressIndicator != null && file.isDirectory()) { // once for dir is cheap enough - myProgressIndicator.checkCanceled(); - myProgressIndicator.setText("Scanning files to index"); + if (oldStuff && file instanceof VirtualFileSystemEntry) { + ((VirtualFileSystemEntry)file).setFileIndexed(true); } } + finally { + FileTypeManagerImpl.cacheFileType(file, null); + } + + if (myProgressIndicator != null && file.isDirectory()) { // once for dir is cheap enough + myProgressIndicator.checkCanceled(); + myProgressIndicator.setText("Scanning files to index"); + } return true; } } @@ -2459,14 +2455,14 @@ public class FileBasedIndexImpl extends FileBasedIndex { private boolean isTooLarge(@NotNull VirtualFile file) { if (SingleRootFileViewProvider.isTooLargeForIntelligence(file)) { - return !myNoLimitCheckTypes.contains(getFileType(file)); + return !myNoLimitCheckTypes.contains(file.getFileType()); } return false; } private boolean isTooLarge(@NotNull VirtualFile file, long contentSize) { if (SingleRootFileViewProvider.isTooLargeForIntelligence(file, contentSize)) { - return !myNoLimitCheckTypes.contains(getFileType(file)); + return !myNoLimitCheckTypes.contains(file.getFileType()); } return false; } @@ -2489,9 +2485,8 @@ public class FileBasedIndexImpl extends FileBasedIndex { PsiFile file = event.getFile(); if (file != null) { VirtualFile virtualFile = file.getVirtualFile(); - FileDocumentManager instance = FileDocumentManager.getInstance(); - Document document = instance.getDocument(virtualFile); - if (document != null && instance.isDocumentUnsaved(document)) { + Document document = myFileDocumentManager.getDocument(virtualFile); + if (document != null && myFileDocumentManager.isDocumentUnsaved(document)) { for(ID psiBackedIndex:myPsiDependentIndices) { myUpToDateIndicesForUnsavedOrTransactedDocuments.remove(psiBackedIndex); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java index 5b3c01eb6d6b..5bb07d82bbcd 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java @@ -59,7 +59,7 @@ public class IndexingStamp { private static final long UNINDEXED_STAMP = -1L; // we don't store trivial "absent" state private static final long INDEX_DATA_OUTDATED_STAMP = -2L; - private static final int VERSION = 9; + private static final int VERSION = 10; private static final ConcurrentHashMap, Long> ourIndexIdToCreationStamp = new ConcurrentHashMap, Long>(); private static volatile long ourLastStamp; // ensure any file index stamp increases diff --git a/platform/platform-api/src/com/intellij/openapi/diff/FileContent.java b/platform/platform-api/src/com/intellij/openapi/diff/FileContent.java index eb930a426fae..b0ac9f0a788f 100644 --- a/platform/platform-api/src/com/intellij/openapi/diff/FileContent.java +++ b/platform/platform-api/src/com/intellij/openapi/diff/FileContent.java @@ -19,8 +19,6 @@ import com.intellij.openapi.editor.Document; import com.intellij.openapi.fileEditor.FileDocumentManager; import com.intellij.openapi.fileEditor.OpenFileDescriptor; import com.intellij.openapi.fileTypes.FileType; -import com.intellij.openapi.fileTypes.FileTypeManager; -import com.intellij.openapi.fileTypes.UnknownFileType; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.vfs.LocalFileSystem; @@ -36,12 +34,12 @@ public class FileContent extends DiffContent { @NotNull private final VirtualFile myFile; private Document myDocument; private final Project myProject; - @Nullable private final FileType myType; + private final FileType myType; public FileContent(Project project, @NotNull VirtualFile file) { myProject = project; myFile = file; - myType = detectType(file); + myType = file.getFileType(); } @Override @@ -66,8 +64,7 @@ public class FileContent extends DiffContent { @Override @Nullable public FileType getContentType() { - FileType type = myFile.getFileType(); - return isUnknown(type) ? myType : type; + return myType; } @Override @@ -78,11 +75,7 @@ public class FileContent extends DiffContent { @Override public boolean isBinary() { - if (myFile.isDirectory()) return false; - if (myType != null && !myType.isBinary()) { - return false; - } - return myFile.getFileType().isBinary(); + return !myFile.isDirectory() && myType.isBinary(); } public static FileContent createFromTempFile(Project project, String name, String ext, @NotNull byte[] content) throws IOException { @@ -102,21 +95,6 @@ public class FileContent extends DiffContent { throw new IOException("Can not create temp file for revision content"); } - @Nullable - static FileType detectType(@NotNull VirtualFile file) { - FileType type = FileTypeManager.getInstance().getFileTypeByFile(file); - if (isUnknown(type)) { - type = FileTypeManager.getInstance().detectFileTypeFromContent(file); - } - // the type is left null intentionally: according to the contract of #getContentType, - // if file type is null it may be taken from another diff content - return isUnknown(type) ? null : type; - } - - private static boolean isUnknown(@NotNull FileType type) { - return type.equals(UnknownFileType.INSTANCE); - } - @NotNull @Override public LineSeparator getLineSeparator() { diff --git a/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/FileAttribute.java b/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/FileAttribute.java index 6f3b59cf7a69..965b3c0de9bf 100644 --- a/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/FileAttribute.java +++ b/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/FileAttribute.java @@ -47,11 +47,15 @@ public class FileAttribute { } public FileAttribute(@NonNls @NotNull String id, int version, boolean fixedSize) { + this(version, fixedSize, id); + boolean added = ourRegisteredIds.add(id); + assert added : "Attribute id='" + id+ "' is not unique"; + } + + private FileAttribute(int version, boolean fixedSize,@NotNull String id) { myId = id; myVersion = version; myFixedSize = fixedSize; - boolean added = ourRegisteredIds.add(id); - assert added : "Attribute id='" + id+ "' is not unique"; } @Nullable @@ -122,4 +126,9 @@ public class FileAttribute { public boolean isFixedSize() { return myFixedSize; } + + @NotNull + public FileAttribute newVersion(int newVersion) { + return new FileAttribute(newVersion, myFixedSize, myId); + } } diff --git a/platform/platform-impl/src/com/intellij/ide/dnd/FileCopyPasteUtil.java b/platform/platform-impl/src/com/intellij/ide/dnd/FileCopyPasteUtil.java index dca050bcd31f..4faddff03bab 100644 --- a/platform/platform-impl/src/com/intellij/ide/dnd/FileCopyPasteUtil.java +++ b/platform/platform-impl/src/com/intellij/ide/dnd/FileCopyPasteUtil.java @@ -16,8 +16,6 @@ package com.intellij.ide.dnd; import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.fileTypes.FileTypeRegistry; -import com.intellij.openapi.fileTypes.UnknownFileType; import com.intellij.openapi.ide.CopyPasteManager; import com.intellij.openapi.util.Condition; import com.intellij.openapi.util.text.StringUtil; @@ -148,9 +146,7 @@ public class FileCopyPasteUtil { if (virtualFile == null) continue; result.add(virtualFile); // detect and store file type for Finder-2-IDEA drag-n-drop - if (!virtualFile.isDirectory() && virtualFile.getFileType() == UnknownFileType.INSTANCE) { - FileTypeRegistry.getInstance().detectFileTypeFromContent(virtualFile); - } + virtualFile.getFileType(); } } return result; diff --git a/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/FileDocumentManagerImpl.java b/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/FileDocumentManagerImpl.java index 54803815c898..1a3159f200a0 100644 --- a/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/FileDocumentManagerImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/fileEditor/impl/FileDocumentManagerImpl.java @@ -175,9 +175,7 @@ public class FileDocumentManagerImpl extends FileDocumentManager implements Appl return null; } if (isBinaryWithoutDecompiler(file)) { - FileType fileType = file.getFileType(); - if (fileType == UnknownFileType.INSTANCE) fileType = FileTypeManager.getInstance().detectFileTypeFromContent(file); - if (fileType.isBinary()) return null; + return null; } final CharSequence text = LoadTextUtil.loadText(file); @@ -563,11 +561,7 @@ public class FileDocumentManagerImpl extends FileDocumentManager implements Appl else if (VirtualFile.PROP_NAME.equals(event.getPropertyName())) { Document document = getCachedDocument(file); if (document != null) { - FileType type = file.getFileType(); - if (type == UnknownFileType.INSTANCE) { - // a file is linked to a document - chances are it is an "unknown text file" now - FileTypeManager.getInstance().detectFileTypeFromContent(file); - } + // a file is linked to a document - chances are it is an "unknown text file" now if (isBinaryWithoutDecompiler(file)) { file.putUserData(DOCUMENT_KEY, null); document.putUserData(FILE_KEY, null); @@ -750,7 +744,7 @@ public class FileDocumentManagerImpl extends FileDocumentManager implements Appl public static boolean recomputeFileTypeIfNecessary(@NotNull VirtualFile virtualFile) { if (virtualFile.getUserData(MUST_RECOMPUTE_FILE_TYPE) != null) { - FileTypeRegistry.getInstance().detectFileTypeFromContent(virtualFile); + virtualFile.getFileType(); virtualFile.putUserData(MUST_RECOMPUTE_FILE_TYPE, null); return true; } @@ -759,25 +753,6 @@ public class FileDocumentManagerImpl extends FileDocumentManager implements Appl @Override public void beforeFileDeletion(@NotNull VirtualFileEvent event) { - /* - if (!event.isFromRefresh()) { - VirtualFile file = event.getFile(); - if (file.getFileSystem() instanceof TempFileSystem) { - return; //hack: this fs fails in getChildren during beforeFileDeletion - } - VfsUtilCore.visitChildrenRecursively(file, new VirtualFileVisitor() { - @Override - public boolean visitFile(@NotNull VirtualFile file) { - Document document = getCachedDocument(file); - if (document != null) { - removeFromUnsaved(document); - } - return true; - } - }); - } - */ - } @Override diff --git a/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java b/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java index 643e22c0d54f..9b3b40e9a8ce 100644 --- a/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java @@ -18,6 +18,7 @@ package com.intellij.openapi.fileTypes.impl; import com.intellij.ide.highlighter.custom.SyntaxTable; import com.intellij.ide.highlighter.custom.impl.ReadFileType; import com.intellij.ide.plugins.PluginManager; +import com.intellij.openapi.Disposable; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.PathManager; import com.intellij.openapi.components.ExportableApplicationComponent; @@ -36,13 +37,16 @@ import com.intellij.openapi.util.*; import com.intellij.openapi.util.io.ByteSequence; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.openapi.vfs.*; +import com.intellij.openapi.vfs.CharsetToolkit; +import com.intellij.openapi.vfs.VFileProperty; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileWithId; +import com.intellij.openapi.vfs.newvfs.FileAttribute; import com.intellij.openapi.vfs.newvfs.FileSystemInterface; import com.intellij.psi.SingleRootFileViewProvider; import com.intellij.testFramework.LightVirtualFile; -import com.intellij.util.ArrayUtil; -import com.intellij.util.PlatformUtils; -import com.intellij.util.Processor; +import com.intellij.util.*; +import com.intellij.util.containers.ConcurrentBitSet; import com.intellij.util.messages.MessageBus; import com.intellij.util.messages.MessageBusConnection; import gnu.trove.THashMap; @@ -54,18 +58,16 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** * @author Yura Cangea */ -public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOMExternalizable, ExportableApplicationComponent { +public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOMExternalizable, ExportableApplicationComponent, Disposable { private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.fileTypes.impl.FileTypeManagerImpl"); private static final int VERSION = 11; private static final Key FILE_TYPE_KEY = Key.create("FILE_TYPE_KEY"); @@ -98,10 +100,10 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME @NonNls private static final String ATTRIBUTE_DEFAULT_EXTENSION = "default_extension"; private static class StandardFileType { - private final FileType fileType; - private final List matchers; + @NotNull private final FileType fileType; + @NotNull private final List matchers; - private StandardFileType(final FileType fileType, final List matchers) { + private StandardFileType(@NotNull FileType fileType, @NotNull List matchers) { this.fileType = fileType; this.matchers = matchers; } @@ -112,6 +114,11 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME @NonNls private static final String[] FILE_TYPES_WITH_PREDEFINED_EXTENSIONS = {"JSP", "JSPX", "DTD", "HTML", "Properties", "XHTML"}; private final SchemesManager mySchemesManager; @NonNls private static final String FILE_SPEC = "$ROOT_CONFIG$/filetypes"; + private final ConcurrentBitSet autoDetectWasRun = new ConcurrentBitSet(); + private final ConcurrentBitSet autoDetectedAsText = new ConcurrentBitSet(); + private final ConcurrentBitSet autoDetectedAsBinary = new ConcurrentBitSet(); + private final AtomicInteger counterAutoDetect = new AtomicInteger(); + private final AtomicLong elapsedAutoDetect = new AtomicLong(); private void initStandardFileTypes() { final FileTypeConsumer consumer = new FileTypeConsumer() { @@ -121,12 +128,12 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME } @Override - public void consume(@NotNull final FileType fileType, final String extensions) { + public void consume(@NotNull final FileType fileType, String extensions) { register(fileType, parse(extensions)); } @Override - public void consume(@NotNull final FileType fileType, final FileNameMatcher... matchers) { + public void consume(@NotNull final FileType fileType, @NotNull final FileNameMatcher... matchers) { register(fileType, new ArrayList(Arrays.asList(matchers))); } @@ -136,7 +143,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME return type != null ? type.fileType : null; } - private void register(final FileType fileType, final List fileNameMatchers) { + private void register(@NotNull FileType fileType, @NotNull List fileNameMatchers) { final StandardFileType type = myStandardFileTypes.get(fileType.getName()); if (type != null) { for (FileNameMatcher matcher : fileNameMatchers) type.matchers.add(matcher); @@ -166,9 +173,6 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME mySchemesManager = schemesManagerFactory.createSchemesManager(FILE_SPEC, new BaseSchemeProcessor() { @Override public AbstractFileType readScheme(@NotNull final Document document) throws InvalidDataException { - if (document == null) { - throw new InvalidDataException(); - } Element root = document.getRootElement(); if (root == null || !ELEMENT_FILETYPE.equals(root.getName())) { throw new InvalidDataException(); @@ -337,35 +341,92 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME fileType = getFileTypeByFileName(file.getNameSequence()); if (fileType != UnknownFileType.INSTANCE) return fileType; - fileType = cachedDetectedFromContent(file); - if (fileType != null) return fileType; + fileType = getOrDetectFromContent(file); - return UnknownFileType.INSTANCE; + return fileType; } - private static FileType cachedDetectedFromContent(@NotNull VirtualFile file) { - return file.getUserData(DETECTED_FROM_CONTENT_FILE_TYPE_KEY); + @NotNull + private FileType getOrDetectFromContent(@NotNull VirtualFile file) { + if (!isDetectable(file)) return UnknownFileType.INSTANCE; + if (file instanceof VirtualFileWithId) { + int id = ((VirtualFileWithId)file).getId(); + if (id < 0) return UnknownFileType.INSTANCE; + if (autoDetectWasRun.get(id)) { + return autoDetectedAsText.get(id) ? FileTypes.PLAIN_TEXT : autoDetectedAsBinary.get(id) ? UnknownFileType.INSTANCE : + ObjectUtils.notNull(file.getUserData(DETECTED_FROM_CONTENT_FILE_TYPE_KEY), FileTypes.PLAIN_TEXT); + } + boolean wasDetectedAsText = false; + boolean wasDetectedAsBinary = false; + boolean wasAutoDetectRun = false; + DataInputStream stream = autoDetectedAttribute.readAttribute(file); + try { + try { + byte status = stream != null ? stream.readByte() : 0; + wasAutoDetectRun = stream != null; + wasDetectedAsText = BitUtil.isSet(status, AUTO_DETECTED_AS_TEXT_FLAG); + wasDetectedAsBinary = BitUtil.isSet(status, AUTO_DETECTED_AS_BINARY_FLAG); + } + finally { + if (stream != null) { + stream.close(); + } + } + } + catch (IOException ignored) { + } + autoDetectWasRun.set(id, wasAutoDetectRun); + autoDetectedAsText.set(id, wasDetectedAsText); + autoDetectedAsBinary.set(id, wasDetectedAsBinary); + if (wasAutoDetectRun && (wasDetectedAsText || wasDetectedAsBinary)) { + return wasDetectedAsText ? FileTypes.PLAIN_TEXT : UnknownFileType.INSTANCE; + } + } + FileType fileType = file.getUserData(DETECTED_FROM_CONTENT_FILE_TYPE_KEY); + // run autodetection + if (fileType == null) { + fileType = detectFromContent(file); + } + + return fileType; } @NotNull @Override public FileType detectFileTypeFromContent(@NotNull VirtualFile file) { - if (file.isDirectory() || !file.isValid() || file.is(VFileProperty.SPECIAL)) { - return UnknownFileType.INSTANCE; - } - FileType fileType = cachedDetectedFromContent(file); - if (fileType == null) { - fileType = detectFromContent(file); - // for empty file there is still hope its type will change - if (file.getLength() != 0) { - file.putUserData(DETECTED_FROM_CONTENT_FILE_TYPE_KEY, fileType); - } - } - return fileType; + return file.getFileType(); } - public static boolean isFileTypeDetectedFromContent(@NotNull VirtualFile file) { - return cachedDetectedFromContent(file) != null; + private volatile FileAttribute autoDetectedAttribute = new FileAttribute("AUTO_DETECTION_CACHE_ATTRIBUTE", 0, true); + private static final int AUTO_DETECTED_AS_TEXT_FLAG = 0x01; + private static final int AUTO_DETECTED_AS_BINARY_FLAG = 0x02; + private void cacheAutoDetectedFileType(@NotNull VirtualFile file, @NotNull FileType fileType) { + DataOutputStream stream = autoDetectedAttribute.writeAttribute(file); + boolean wasAutodetectedAsText = fileType == FileTypes.PLAIN_TEXT; + boolean wasAutodetectedAsBinary = fileType == FileTypes.UNKNOWN; + try { + try { + byte b = (byte)((wasAutodetectedAsText ? AUTO_DETECTED_AS_TEXT_FLAG : 0) | + (wasAutodetectedAsBinary ? AUTO_DETECTED_AS_BINARY_FLAG : 0)); + stream.writeByte(b); + } + finally { + stream.close(); + } + } + catch (IOException e) { + LOG.error(e); + } + if (file instanceof VirtualFileWithId) { + int id = Math.abs(((VirtualFileWithId)file).getId()); + autoDetectWasRun.set(id); + autoDetectedAsText.set(id, wasAutodetectedAsText); + autoDetectedAsBinary.set(id, wasAutodetectedAsBinary); + if (wasAutodetectedAsText || wasAutodetectedAsBinary) { + return; + } + } + file.putUserData(DETECTED_FROM_CONTENT_FILE_TYPE_KEY, fileType); } @Override @@ -386,20 +447,19 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME private static final AtomicInteger DETECTED_COUNT = new AtomicInteger(); private static final int DETECT_BUFFER_SIZE = 8192; + private static boolean isDetectable(@NotNull final VirtualFile file) { + if (file.isDirectory() || !file.isValid() || file.is(VFileProperty.SPECIAL) || file.getLength() == 0) { + // for empty file there is still hope its type will change + return false; + } + return file.getFileSystem() instanceof FileSystemInterface && !SingleRootFileViewProvider.isTooLargeForContentLoading(file); + } + @NotNull - private static FileType detectFromContent(@NotNull final VirtualFile file) { + private FileType detectFromContent(@NotNull final VirtualFile file) { + long start = System.currentTimeMillis(); try { - final long length = file.getLength(); - if (length == 0) return UnknownFileType.INSTANCE; - - final VirtualFileSystem fileSystem = file.getFileSystem(); - if (!(fileSystem instanceof FileSystemInterface)) return UnknownFileType.INSTANCE; - - if (SingleRootFileViewProvider.isTooLargeForContentLoading(file)) { - return UnknownFileType.INSTANCE; - } - - final InputStream inputStream = ((FileSystemInterface)fileSystem).getInputStream(file); + final InputStream inputStream = ((FileSystemInterface)file.getFileSystem()).getInputStream(file); final Ref result; try { result = new Ref(UnknownFileType.INSTANCE); @@ -410,7 +470,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME CharSequence text; if (isText) { byte[] bytes = Arrays.copyOf(byteSequence.getBytes(), byteSequence.getLength()); - text = LoadTextUtil.getTextByBinaryPresentation(bytes, file); + text = LoadTextUtil.getTextByBinaryPresentation(bytes, file, true, true, UnknownFileType.INSTANCE); } else { text = null; @@ -443,11 +503,14 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME if (LOG.isDebugEnabled()) { LOG.debug(file + "; type=" + fileType.getDescription() + "; " + DETECTED_COUNT.incrementAndGet()); } + + cacheAutoDetectedFileType(file, fileType); + counterAutoDetect.incrementAndGet(); + long elapsed = System.currentTimeMillis() - start; + elapsedAutoDetect.addAndGet(elapsed); + return fileType; } - catch (FileNotFoundException e) { - return UnknownFileType.INSTANCE; - } catch (IOException e) { LOG.info(e); return UnknownFileType.INSTANCE; @@ -614,11 +677,20 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME return mySchemesManager; } + private final AtomicInteger fileTypeChangedCount = new AtomicInteger(); @Override public void fireFileTypesChanged() { + clearCaches(); myMessageBus.syncPublisher(TOPIC).fileTypesChanged(new FileTypeEvent(this)); } + private void clearCaches() { + autoDetectedAttribute = autoDetectedAttribute.newVersion(fileTypeChangedCount.incrementAndGet()); + autoDetectWasRun.clear(); + autoDetectedAsText.clear(); + autoDetectedAsBinary.clear(); + } + private final Map myAdapters = new HashMap(); @Override @@ -713,10 +785,11 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME addIgnore("*.rbc"); } myIgnoredFileCache.clearCache(); + fileTypeChangedCount.set(JDOMExternalizer.readInteger(parentNode, "fileTypeChangedCounter", 0)); + autoDetectedAttribute = autoDetectedAttribute.newVersion(fileTypeChangedCount.get()); } - private void readGlobalMappings(final Element e) { - + private void readGlobalMappings(@NotNull Element e) { List> associations = AbstractFileType.readAssociations(e); for (Pair association : associations) { @@ -744,8 +817,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME } } - private void readMappingsForFileType(final Element e, FileType type) { - + private void readMappingsForFileType(@NotNull Element e, FileType type) { List> associations = AbstractFileType.readAssociations(e); for (Pair association : associations) { @@ -757,10 +829,9 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME for (Trinity removedAssociation : removedAssociations) { removeAssociation(type, removedAssociation.getFirst(), false); } - } - private void addIgnore(@NonNls final String ignoreMask) { + private void addIgnore(@NonNls @NotNull String ignoreMask) { myIgnoredPatterns.addIgnoreMask(ignoreMask); } @@ -784,7 +855,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME } } - private static int getVersion(final Element node) { + private static int getVersion(@NotNull Element node) { final String verString = node.getAttributeValue(ATTRIBUTE_VERSION); if (verString == null) return 0; try { @@ -815,6 +886,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME for (FileType type : fileTypes) { writeExtensionsMap(map, type, true); } + JDOMExternalizer.write(parentNode, "fileTypeChangedCounter", fileTypeChangedCount.get()); } private void writeExtensionsMap(final Element map, final FileType type, boolean specifyTypeName) { @@ -865,10 +937,11 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME // ------------------------------------------------------------------------- @Nullable - private FileType getFileTypeByName(String name) { + private FileType getFileTypeByName(@NotNull String name) { return mySchemesManager.findSchemeByName(name); } + @NotNull private static List parse(@NonNls String semicolonDelimited) { if (semicolonDelimited == null) return Collections.emptyList(); StringTokenizer tokenizer = new StringTokenizer(semicolonDelimited, FileTypeConsumer.EXTENSION_DELIMITER, false); @@ -882,7 +955,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME /** * Registers a standard file type. Doesn't notifyListeners any change events. */ - private void registerFileTypeWithoutNotification(FileType fileType, List matchers) { + private void registerFileTypeWithoutNotification(@NotNull FileType fileType, @NotNull List matchers) { mySchemesManager.addNewScheme(fileType, true); for (FileNameMatcher matcher : matchers) { myPatternsTable.addAssociation(matcher, fileType); @@ -895,7 +968,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME } - private void bindUnresolvedMappings(FileType fileType) { + private void bindUnresolvedMappings(@NotNull FileType fileType) { for (FileNameMatcher matcher : new THashSet(myUnresolvedMappings.keySet())) { String name = myUnresolvedMappings.get(matcher); if (Comparing.equal(name, fileType.getName())) { @@ -914,7 +987,6 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME } // returns true if at least one standard file type has been read - @SuppressWarnings({"EmptyCatchBlock"}) private boolean loadAllFileTypes() { Collection collection = mySchemesManager.loadSchemes(); @@ -926,16 +998,16 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME } return res; - } - private FileType loadFileType(final ReadFileType readFileType) { - return loadFileType(readFileType.getElement(), false, mySchemesManager.isShared(readFileType) ? readFileType.getExternalInfo() : null, + private FileType loadFileType(@NotNull ReadFileType readFileType) { + ExternalInfo info = mySchemesManager.isShared(readFileType) ? readFileType.getExternalInfo() : null; + return loadFileType(readFileType.getElement(), false, info, true, readFileType.getExternalInfo().getCurrentFileName()); } - private FileType loadFileType(Element typeElement, boolean isDefaults, final ExternalInfo info, boolean ignoreExisting, String fileName) { + private FileType loadFileType(@NotNull Element typeElement, boolean isDefaults, final ExternalInfo info, boolean ignoreExisting, String fileName) { String fileTypeName = typeElement.getAttributeValue(ATTRIBUTE_NAME); String fileTypeDescr = typeElement.getAttributeValue(ATTRIBUTE_DESCRIPTION); String iconPath = typeElement.getAttributeValue(ATTRIBUTE_ICON); @@ -1007,7 +1079,8 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME return StringUtil.join(list, FileTypeConsumer.EXTENSION_DELIMITER); } - private static FileType loadCustomFile(final Element typeElement, ExternalInfo info, String fileName) { + @NotNull + private static FileType loadCustomFile(@NotNull Element typeElement, ExternalInfo info, String fileName) { FileType type = null; Element element = typeElement.getChild(AbstractFileType.ELEMENT_HIGHLIGHTING); @@ -1161,4 +1234,9 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements NamedJDOME Map> getRemovedMappings() { return myRemovedMappings; } + + @Override + public void dispose() { + LOG.info("FileTypeManager: "+ counterAutoDetect +" auto-detected files\nElapsed time on auto-detect: "+elapsedAutoDetect+" ms"); + } } diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypeManagerTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypeManagerTest.java index 57828d0e81a6..d0289668391a 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypeManagerTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypeManagerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2012 JetBrains s.r.o. + * Copyright 2000-2014 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,28 +15,12 @@ */ package com.intellij.openapi.fileTypes; -import com.intellij.openapi.vfs.VfsUtil; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.psi.PsiFile; -import com.intellij.psi.PsiManager; -import com.intellij.psi.impl.PsiManagerEx; import com.intellij.testFramework.fixtures.LightPlatformCodeInsightFixtureTestCase; import java.io.IOException; public class FileTypeManagerTest extends LightPlatformCodeInsightFixtureTestCase { - public void testAutoDetectTextFileFromContents() throws IOException { - VirtualFile vFile = myFixture.getTempDirFixture().createFile("test.xxxxxxxx"); - VfsUtil.saveText(vFile, "text"); - - FileType type = vFile.getFileType(); - assertEquals(UnknownFileType.INSTANCE, type); - - PsiFile psiFile = ((PsiManagerEx)PsiManager.getInstance(myFixture.getProject())).getFileManager().findFile(vFile); // autodetect text file if needed - assertNotNull(psiFile); - assertEquals(PlainTextFileType.INSTANCE, vFile.getFileType()); - } - public void testIgnoredFiles() throws IOException { VirtualFile vFile = myFixture.getTempDirFixture().createFile(".svn", ""); assertTrue(FileTypeManager.getInstance().isFileIgnored(vFile)); diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypesTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypesTest.java index 4dd03667a52d..42e932c6aa93 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypesTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/FileTypesTest.java @@ -23,10 +23,13 @@ import com.intellij.openapi.fileTypes.impl.FileTypeAssocTable; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.LocalFileSystem; +import com.intellij.openapi.vfs.VfsUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiBinaryFile; import com.intellij.psi.PsiFile; +import com.intellij.psi.PsiManager; import com.intellij.psi.PsiPlainTextFile; +import com.intellij.psi.impl.PsiManagerEx; import com.intellij.testFramework.PlatformTestCase; import com.intellij.util.PatternUtil; @@ -184,7 +187,7 @@ public class FileTypesTest extends PlatformTestCase { FileUtil.writeToFile(file, "xxx xxx xxx xxx"); VirtualFile virtualFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(file); assertNotNull(virtualFile); - assertEquals(FileTypes.UNKNOWN, virtualFile.getFileType()); + assertEquals(PlainTextFileType.INSTANCE, virtualFile.getFileType()); } public void testAutoDetectEmptyFile() throws IOException { @@ -205,4 +208,29 @@ public class FileTypesTest extends PlatformTestCase { assertTrue(after.isValid()); assertTrue(after instanceof PsiPlainTextFile); } + + public void testAutoDetectTextFileFromContents() throws IOException { + File dir = createTempDirectory(); + VirtualFile vDir = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(dir); + VirtualFile vFile = vDir.createChildData(this, "test.xxxxxxxx"); + VfsUtil.saveText(vFile, "text"); + + assertEquals(PlainTextFileType.INSTANCE, vFile.getFileType()); // type autodetected during indexing + + PsiFile psiFile = ((PsiManagerEx)PsiManager.getInstance(getProject())).getFileManager().findFile(vFile); // autodetect text file if needed + assertNotNull(psiFile); + assertEquals(PlainTextFileType.INSTANCE, psiFile.getFileType()); + } + + public void testAutoDetectTextFileEvenOutsideTheProject() throws IOException { + File d = createTempDirectory(); + File f = new File(d, "xx.asfdasdfas"); + FileUtil.writeToFile(f, "asdasdasdfafds"); + VirtualFile vFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(f); + + assertEquals(PlainTextFileType.INSTANCE, vFile.getFileType()); + + //FileBasedIndex.getInstance().ensureUpToDate(IdIndex.NAME, myProject, GlobalSearchScope.allScope(myProject)); + //assertEquals(PlainTextFileType.INSTANCE, vFile.getFileType()); // type autodetected during indexing + } } diff --git a/platform/util/src/com/intellij/util/containers/ConcurrentBitSet.java b/platform/util/src/com/intellij/util/containers/ConcurrentBitSet.java index 38e44df366ab..cc134cd1be95 100644 --- a/platform/util/src/com/intellij/util/containers/ConcurrentBitSet.java +++ b/platform/util/src/com/intellij/util/containers/ConcurrentBitSet.java @@ -17,6 +17,9 @@ package com.intellij.util.containers; import org.jetbrains.annotations.NotNull; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; import java.util.concurrent.atomic.AtomicLongArray; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -30,6 +33,9 @@ import java.util.concurrent.atomic.AtomicReferenceArray; * @see java.util.BitSet */ public class ConcurrentBitSet { + public ConcurrentBitSet() { + } + /** * An array of 32 longword vectors. * Vector at index "i" has length of (1 << i) long words. @@ -97,9 +103,10 @@ public class ConcurrentBitSet { * Sets the bit at the specified index to {@code true}. * * @param bitIndex a bit index + * @return previous value * @throws IndexOutOfBoundsException if the specified index is negative */ - public void set(int bitIndex) { + public boolean set(int bitIndex) { if (bitIndex < 0) { throw new IndexOutOfBoundsException("bitIndex < 0: " + bitIndex); } @@ -109,11 +116,14 @@ public class ConcurrentBitSet { int wordIndexInArray = wordIndexInArray(bitIndex); long word; long newWord; + boolean previousBit; do { word = array.get(wordIndexInArray); + previousBit = (word & (1L << bitIndex)) != 0; newWord = word | (1L << bitIndex); } while (!array.compareAndSet(wordIndexInArray, word, newWord)); + return previousBit; } /** @@ -137,8 +147,9 @@ public class ConcurrentBitSet { * * @param bitIndex the index of the bit to be cleared * @throws IndexOutOfBoundsException if the specified index is negative + * @return previous value */ - public void clear(int bitIndex) { + public boolean clear(int bitIndex) { if (bitIndex < 0) { throw new IndexOutOfBoundsException("bitIndex < 0: " + bitIndex); } @@ -148,11 +159,14 @@ public class ConcurrentBitSet { int wordIndexInArray = wordIndexInArray(bitIndex); long word; long newWord; + boolean previousBit; do { word = array.get(wordIndexInArray); + previousBit = (word & (1L << bitIndex)) != 0; newWord = word & ~ (1L << bitIndex); } while (!array.compareAndSet(wordIndexInArray, word, newWord)); + return previousBit; } @NotNull @@ -408,4 +422,60 @@ public class ConcurrentBitSet { b.append('}'); return b.toString(); } + + @NotNull + public long[] toLongArray() { + int bits = size(); + long[] result = new long[bits/BITS_PER_WORD]; + int i = 0; + for (int b=0; b