diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/FileTypeIdIndexer.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/FileTypeIdIndexer.java index 5c1246c7a50b..33bfd11cf0f3 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/FileTypeIdIndexer.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/FileTypeIdIndexer.java @@ -16,15 +16,12 @@ package com.intellij.psi.impl.cache.impl.id; -import com.intellij.util.indexing.DataIndexer; -import com.intellij.util.indexing.FileContent; - /** + * @deprecated left for compatibility, use IdIndexer interface instead. To be removed in IntelliJ 2018.3 + * * @author Eugene Zhuravlev * Date: Jan 16, 2008 */ -public abstract class FileTypeIdIndexer implements DataIndexer { - public int getVersion() { - return 1; - } +@Deprecated +public abstract class FileTypeIdIndexer implements IdIndexer { } diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java index c1eec437bdd3..8cc20de4e4b0 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java @@ -74,7 +74,7 @@ public class IdIndex extends FileBasedIndexExtension { @Override @NotNull public Map map(@NotNull final FileContent inputData) { - final FileTypeIdIndexer indexer = IdTableBuilding.getFileTypeIndexer(inputData.getFileType()); + final IdIndexer indexer = IdTableBuilding.getFileTypeIndexer(inputData.getFileType()); if (indexer != null) { return indexer.map(inputData); } diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexer.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexer.java new file mode 100644 index 000000000000..51544a1f382c --- /dev/null +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexer.java @@ -0,0 +1,14 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.psi.impl.cache.impl.id; + +import com.intellij.util.indexing.DataIndexer; +import com.intellij.util.indexing.FileContent; + +/** + * @author traff + */ +public interface IdIndexer extends DataIndexer { + default int getVersion() { + return 1; + } +} diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexers.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexers.java index db1fcf6d91b1..e6206ba1e257 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexers.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexers.java @@ -21,7 +21,7 @@ import com.intellij.openapi.fileTypes.FileTypeExtension; /** * @author yole */ -public class IdIndexers extends FileTypeExtension { +public class IdIndexers extends FileTypeExtension { public static IdIndexers INSTANCE = new IdIndexers(); private IdIndexers() { diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java index 0037cdb42e94..0bfe43a88621 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java @@ -47,10 +47,10 @@ public class IdTableBuilding { void run(CharSequence chars, @Nullable char[] charsArray, int start, int end); } - private static final Map ourIdIndexers = new HashMap<>(); + private static final Map ourIdIndexers = new HashMap<>(); @Deprecated - public static void registerIdIndexer(@NotNull FileType fileType, FileTypeIdIndexer indexer) { + public static void registerIdIndexer(@NotNull FileType fileType, IdIndexer indexer) { ourIdIndexers.put(fileType, indexer); } @@ -60,14 +60,14 @@ public class IdTableBuilding { @Nullable - public static FileTypeIdIndexer getFileTypeIndexer(FileType fileType) { - final FileTypeIdIndexer idIndexer = ourIdIndexers.get(fileType); + public static IdIndexer getFileTypeIndexer(FileType fileType) { + final IdIndexer idIndexer = ourIdIndexers.get(fileType); if (idIndexer != null) { return idIndexer; } - final FileTypeIdIndexer extIndexer = IdIndexers.INSTANCE.forFileType(fileType); + final IdIndexer extIndexer = IdIndexers.INSTANCE.forFileType(fileType); if (extIndexer != null) { return extIndexer; } diff --git a/platform/lang-impl/src/com/intellij/index/PrebuiltIndex.kt b/platform/lang-impl/src/com/intellij/index/PrebuiltIndex.kt new file mode 100644 index 000000000000..451e83195846 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/index/PrebuiltIndex.kt @@ -0,0 +1,113 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.index + +import com.google.common.hash.HashCode +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.PathManager +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.util.io.FileUtil +import com.intellij.psi.stubs.FileContentHashing +import com.intellij.psi.stubs.HashCodeDescriptor +import com.intellij.psi.stubs.PrebuiltStubsProviderBase +import com.intellij.util.indexing.FileContent +import com.intellij.util.indexing.IndexInfrastructure +import com.intellij.util.io.DataExternalizer +import com.intellij.util.io.PersistentHashMap +import java.io.File +import java.io.FileFilter +import java.io.IOException + +/** + * @author traff + */ +abstract class PrebuiltIndexProviderBase : Disposable { + private val myFileContentHashing = FileContentHashing() + private var myPrebuiltIndexStorage: PersistentHashMap? = null + + protected abstract val dirName: String + protected abstract val indexName: String + protected abstract val indexExternalizer: DataExternalizer + + companion object { + private val LOG = Logger.getInstance("#com.intellij.index.PrebuiltIndexProviderBase") + } + + init { + init() + } + + internal fun init() { + var indexesRoot = findPrebuiltIndicesRoot() + try { + if (indexesRoot != null) { + // we should copy prebuilt indexes to a writable folder + indexesRoot = copyPrebuiltIndicesToIndexRoot(indexesRoot) + // otherwise we can get access denied error, because persistent hash map opens file for read and write + + myPrebuiltIndexStorage = openIndexStorage(indexesRoot) + + LOG.info("Using prebuilt $indexName from " + myPrebuiltIndexStorage!!.baseFile.absolutePath) + } + } + catch (e: Exception) { + myPrebuiltIndexStorage = null + LOG.warn("Prebuilt indices can't be loaded at " + indexesRoot!!, e) + } + } + + fun get(fileContent: FileContent): Value? { + if (myPrebuiltIndexStorage != null) { + val hashCode = myFileContentHashing.hashString(fileContent) + try { + return myPrebuiltIndexStorage!!.get(hashCode) + } + catch (e: Exception) { + LOG.error("Error reading prebuilt stubs from " + myPrebuiltIndexStorage!!.baseFile.path, e) + myPrebuiltIndexStorage = null + } + } + return null + } + + open fun openIndexStorage(indexesRoot: File): PersistentHashMap? { + return object : PersistentHashMap( + File(indexesRoot, indexName + ".input"), + HashCodeDescriptor.instance, + indexExternalizer) { + override fun isReadOnly(): Boolean { + return true + } + } + } + + @Throws(IOException::class) + private fun copyPrebuiltIndicesToIndexRoot(prebuiltIndicesRoot: File): File { + val indexRoot = File(IndexInfrastructure.getPersistentIndexRoot(), "prebuilt/" + dirName) + + FileUtil.copyDir(prebuiltIndicesRoot, indexRoot, FileFilter { f -> f.nameWithoutExtension == indexName }) + + return indexRoot + } + + private fun findPrebuiltIndicesRoot(): File? { + val path: String? = System.getProperty(PrebuiltStubsProviderBase.PREBUILT_INDICES_PATH_PROPERTY) + if (path != null && File(path).exists()) { + return File(path) + } + val f = indexRoot() + return if (f.exists()) f else null + } + + open fun indexRoot(): File = File(PathManager.getHomePath(), "index/$dirName") // compiled binary + + override fun dispose() { + if (myPrebuiltIndexStorage != null) { + try { + myPrebuiltIndexStorage!!.close() + } + catch (e: IOException) { + LOG.error(e) + } + } + } +} diff --git a/platform/lang-impl/src/com/intellij/index/PrebuiltIndexAwareIdIndexer.kt b/platform/lang-impl/src/com/intellij/index/PrebuiltIndexAwareIdIndexer.kt new file mode 100644 index 000000000000..03c0b30dded1 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/index/PrebuiltIndexAwareIdIndexer.kt @@ -0,0 +1,57 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.index + +import com.intellij.psi.impl.cache.impl.id.IdIndexEntry +import com.intellij.psi.impl.cache.impl.id.IdIndexer +import com.intellij.util.indexing.FileContent +import com.intellij.util.io.DataExternalizer +import com.intellij.util.io.DataInputOutputUtil +import java.io.DataInput +import java.io.DataOutput + +/** + * @author traff + */ + +abstract class PrebuiltIndexAwareIdIndexer : PrebuiltIndexProviderBase>(), IdIndexer { + companion object { + val ID_INDEX_FILE_NAME = "id-index" + } + + override val indexName = ID_INDEX_FILE_NAME + + override val indexExternalizer = IdIndexMapDataExternalizer() + + override fun map(inputData: FileContent): Map { + val map = get(inputData) + return if (map != null) { + map + } + else { + idIndexMap(inputData) + } + } + + abstract fun idIndexMap(inputData: FileContent): Map +} + +class IdIndexMapDataExternalizer : DataExternalizer> { + override fun save(out: DataOutput, value: Map) { + DataInputOutputUtil.writeINT(out, value.size) + for (e in value.entries) { + DataInputOutputUtil.writeINT(out, e.key.wordHashCode) + DataInputOutputUtil.writeINT(out, e.value) + } + } + + override fun read(`in`: DataInput): Map { + val size = DataInputOutputUtil.readINT(`in`) + val map = HashMap() + for (i in 0..size) { + val wordHash = DataInputOutputUtil.readINT(`in`) + val value= DataInputOutputUtil.readINT(`in`) + map.put(IdIndexEntry(wordHash), value) + } + return map + } +} diff --git a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexImpl.java index 7ff39fbd1792..290e218d0235 100644 --- a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexImpl.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Comparator; public class IdIndexImpl extends IdIndex implements CustomInputsIndexFileBasedIndexExtension { private static final ThreadLocalCachedIntArray spareBufferLocal = new ThreadLocalCachedIntArray(); @@ -48,7 +47,7 @@ public class IdIndexImpl extends IdIndex implements CustomInputsIndexFileBasedIn int version = super.getVersion(); for(FileType fileType:types) { if (!isIndexable(fileType)) continue; - FileTypeIdIndexer indexer = IdTableBuilding.getFileTypeIndexer(fileType); + IdIndexer indexer = IdTableBuilding.getFileTypeIndexer(fileType); if (indexer == null) continue; version = version * 31 + (indexer.getVersion() ^ indexer.getClass().getName().hashCode()); } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt index ff516547ee18..b142483f20e9 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt +++ b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt @@ -17,15 +17,13 @@ package com.intellij.psi.stubs import com.google.common.hash.HashCode import com.google.common.hash.Hashing -import com.intellij.openapi.Disposable +import com.intellij.index.PrebuiltIndexProviderBase import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.application.PathManager import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.fileTypes.FileTypeExtension import com.intellij.openapi.util.Disposer import com.intellij.openapi.util.io.FileUtil import com.intellij.util.indexing.FileContent -import com.intellij.util.indexing.IndexInfrastructure import com.intellij.util.io.DataExternalizer import com.intellij.util.io.DataInputOutputUtil import com.intellij.util.io.KeyDescriptor @@ -35,7 +33,6 @@ import org.jetbrains.annotations.TestOnly import java.io.DataInput import java.io.DataOutput import java.io.File -import java.io.IOException /** * @author traff @@ -53,7 +50,7 @@ interface PrebuiltStubsProvider { class FileContentHashing { private val hashing = Hashing.sha256() - fun hashString(fileContent: FileContent) = hashing.hashBytes(fileContent.content) + fun hashString(fileContent: FileContent) = hashing.hashBytes(fileContent.content)!! } @@ -90,17 +87,15 @@ class StubTreeExternalizer : DataExternalizer { override fun read(`in`: DataInput) = SerializedStubTree(`in`) } -abstract class PrebuiltStubsProviderBase : PrebuiltStubsProvider, Disposable { - private val myFileContentHashing = FileContentHashing() - private var myPrebuiltStubsStorage: PersistentHashMap? = null +abstract class PrebuiltStubsProviderBase : PrebuiltIndexProviderBase(), PrebuiltStubsProvider { + private var mySerializationManager: SerializationManagerImpl? = null protected abstract val stubVersion: Int - protected abstract val name: String - init { - init() - } + override val indexName = SDK_STUBS_STORAGE_NAME + + override val indexExternalizer = StubTreeExternalizer() companion object { val PREBUILT_INDICES_PATH_PROPERTY = "prebuilt_indices_path" @@ -108,100 +103,39 @@ abstract class PrebuiltStubsProviderBase : PrebuiltStubsProvider, Disposable { private val LOG = Logger.getInstance("#com.intellij.psi.stubs.PrebuiltStubsProviderBase") } - internal fun init() { - var indexesRoot = findPrebuiltIndicesRoot() - try { - if (indexesRoot != null) { - // we should copy prebuilt indexes to a writable folder - indexesRoot = copyPrebuiltIndicesToIndexRoot(indexesRoot) - // otherwise we can get access denied error, because persistent hash map opens file for read and write + override fun openIndexStorage(indexesRoot: File): PersistentHashMap? { + val versionInFile = FileUtil.loadFile(File(indexesRoot, indexName + ".version")) - val versionInFile = FileUtil.loadFile(File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".version")) + if (Integer.parseInt(versionInFile) == stubVersion) { + mySerializationManager = SerializationManagerImpl(File(indexesRoot, indexName + ".names")) - if (Integer.parseInt(versionInFile) == stubVersion) { - myPrebuiltStubsStorage = object : PersistentHashMap( - File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".input"), - HashCodeDescriptor.instance, - StubTreeExternalizer()) { - override fun isReadOnly(): Boolean { - return true - } - } + Disposer.register(ApplicationManager.getApplication(), mySerializationManager!!) - mySerializationManager = SerializationManagerImpl(File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".names")) - - Disposer.register(ApplicationManager.getApplication(), mySerializationManager!!) - - LOG.info("Using prebuilt stubs from " + myPrebuiltStubsStorage!!.baseFile.absolutePath) - } - else { - LOG.error("Prebuilt stubs version mismatch: $versionInFile, current version is $stubVersion") - } - } + return super.openIndexStorage(indexesRoot) } - catch (e: Exception) { - myPrebuiltStubsStorage = null - LOG.warn("Prebuilt stubs can't be loaded at " + indexesRoot!!, e) + else { + LOG.error("Prebuilt stubs version mismatch: $versionInFile, current version is $stubVersion") + return null } - } override fun findStub(fileContent: FileContent): Stub? { - if (myPrebuiltStubsStorage != null) { - val hashCode = myFileContentHashing.hashString(fileContent) - var stub: Stub? = null - try { - val stubTree = myPrebuiltStubsStorage!!.get(hashCode) - if (stubTree != null) { - stub = stubTree.getStub(false, mySerializationManager!!) - } - } - catch (e: SerializerNotFoundException) { - LOG.error("Can't deserialize stub tree", e) - } - catch (e: Exception) { - LOG.error("Error reading prebuilt stubs from " + myPrebuiltStubsStorage!!.baseFile.path, e) - myPrebuiltStubsStorage = null - stub = null + var stub: Stub? = null + try { + val stubTree = get(fileContent) + if (stubTree != null) { + stub = stubTree.getStub(false, mySerializationManager!!) } + } + catch (e: SerializerNotFoundException) { + LOG.error("Can't deserialize stub tree", e) + } - if (stub != null) { - return stub - } + if (stub != null) { + return stub } return null } - - override fun dispose() { - if (myPrebuiltStubsStorage != null) { - try { - myPrebuiltStubsStorage!!.close() - } - catch (e: IOException) { - LOG.error(e) - } - } - } - - @Throws(IOException::class) - private fun copyPrebuiltIndicesToIndexRoot(prebuiltIndicesRoot: File): File { - val indexRoot = File(IndexInfrastructure.getPersistentIndexRoot(), "prebuilt/" + name) - - FileUtil.copyDir(prebuiltIndicesRoot, indexRoot) - - return indexRoot - } - - private fun findPrebuiltIndicesRoot(): File? { - val path: String? = System.getProperty(PREBUILT_INDICES_PATH_PROPERTY) - if (path != null && File(path).exists()) { - return File(path) - } - val f = indexRoot() - return if (f.exists()) f else null - } - - open fun indexRoot():File = File(PathManager.getHomePath(), "index/$name") // compiled binary } @TestOnly diff --git a/platform/platform-resources/src/META-INF/LangExtensionPoints.xml b/platform/platform-resources/src/META-INF/LangExtensionPoints.xml index 58925216a26e..674b722410d9 100644 --- a/platform/platform-resources/src/META-INF/LangExtensionPoints.xml +++ b/platform/platform-resources/src/META-INF/LangExtensionPoints.xml @@ -780,7 +780,7 @@ - + diff --git a/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy index 60831b7367b2..141c4d8d9f0a 100644 --- a/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy +++ b/python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy @@ -43,7 +43,7 @@ abstract class PyCharmPropertiesBase extends ProductProperties { } } - new PyPrebuiltStubsGenerator().generateResources(context) + new PyPrebuiltIndicesGenerator().generateResources(context) context.ant.copy(todir: "$targetDirectory/index", failonerror: true) { fileset(dir: "$context.paths.temp/index", erroronmissingdir: true) { @@ -58,7 +58,7 @@ abstract class PyCharmPropertiesBase extends ProductProperties { } } -class PyPrebuiltStubsGenerator implements ResourcesGenerator { +class PyPrebuiltIndicesGenerator implements ResourcesGenerator { @Override File generateResources(BuildContext context) { CompilationTasks.create(context).compileModules(["python-community-tools"]) @@ -77,8 +77,8 @@ class PyPrebuiltStubsGenerator implements ResourcesGenerator { def outputPath = "$context.paths.temp/index" - context.ant.java(classname: "com.jetbrains.python.tools.PyPrebuiltStubsGeneratorKt", fork: true) { - jvmarg(line: "-ea -Xmx1500m") + context.ant.java(classname: "com.jetbrains.python.tools.PyPrebuiltIndicesGeneratorKt", fork: true) { + jvmarg(line: "-ea -Xmx1000m") arg(value: zipPath) arg(value: outputPath) classpath { diff --git a/python/src/com/jetbrains/python/PyIdIndexer.java b/python/src/com/jetbrains/python/PyIdIndexer.java deleted file mode 100644 index 6abd33f25a80..000000000000 --- a/python/src/com/jetbrains/python/PyIdIndexer.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2000-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jetbrains.python; - -import com.intellij.lexer.Lexer; -import com.intellij.psi.impl.cache.impl.OccurrenceConsumer; -import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer; -import com.jetbrains.python.lexer.PythonLexer; - -/** - * @author traff - */ -public class PyIdIndexer extends LexerBasedIdIndexer { - @Override - public Lexer createLexer(OccurrenceConsumer consumer) { - return createIndexingLexer(consumer); - } - - public static Lexer createIndexingLexer(OccurrenceConsumer consumer) { - return new PyFilterLexer(new PythonLexer(), consumer); - } -} diff --git a/python/src/com/jetbrains/python/PyIdIndexer.kt b/python/src/com/jetbrains/python/PyIdIndexer.kt new file mode 100644 index 000000000000..4ea5fefe6a10 --- /dev/null +++ b/python/src/com/jetbrains/python/PyIdIndexer.kt @@ -0,0 +1,36 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python + +import com.intellij.index.PrebuiltIndexAwareIdIndexer +import com.intellij.lexer.Lexer +import com.intellij.psi.impl.cache.impl.OccurrenceConsumer +import com.intellij.psi.impl.cache.impl.id.IdIndexEntry +import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer +import com.intellij.util.indexing.FileContent +import com.jetbrains.python.lexer.PythonLexer +import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider + +/** + * @author traff + */ +class PyIdIndexer : PrebuiltIndexAwareIdIndexer() { + private val myIndexer = MyPyIdIndexer() + + override val dirName: String = PyPrebuiltStubsProvider.NAME + + override fun idIndexMap(inputData: FileContent): Map { + return myIndexer.map(inputData) + } + + companion object { + fun createIndexingLexer(consumer: OccurrenceConsumer): Lexer { + return PyFilterLexer(PythonLexer(), consumer) + } + } + + private class MyPyIdIndexer : LexerBasedIdIndexer() { + override fun createLexer(consumer: OccurrenceConsumer): Lexer { + return createIndexingLexer(consumer) + } + } +} \ No newline at end of file diff --git a/python/src/com/jetbrains/python/PyTodoIndexer.java b/python/src/com/jetbrains/python/PyTodoIndexer.java index df05756e2bd4..5d11bd8dc2c0 100644 --- a/python/src/com/jetbrains/python/PyTodoIndexer.java +++ b/python/src/com/jetbrains/python/PyTodoIndexer.java @@ -25,6 +25,6 @@ import com.intellij.psi.impl.cache.impl.todo.LexerBasedTodoIndexer; public class PyTodoIndexer extends LexerBasedTodoIndexer { @Override public Lexer createLexer(OccurrenceConsumer consumer) { - return PyIdIndexer.createIndexingLexer(consumer); + return PyIdIndexer.Companion.createIndexingLexer(consumer); } } diff --git a/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java b/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java index daa6759ef4b0..f85c7795ffe9 100644 --- a/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java +++ b/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java @@ -34,7 +34,7 @@ public class PyPrebuiltStubsProvider extends PrebuiltStubsProviderBase { @NotNull @Override - protected String getName() { + protected String getDirName() { return NAME; } } diff --git a/python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt b/python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt index c51fb930535b..a9489fb8d80a 100644 --- a/python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt +++ b/python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt @@ -90,13 +90,13 @@ class PyProjectSdkStubsGenerator : ProjectSdkStubsGenerator() { override fun createSdkProducer(sdkPath: String) = createPythonSdkProducer(sdkPath) - override fun createStubsGenerator() = PyStubsGenerator() + override fun createStubsGenerator(stubsFilePath: String) = PyStubsGenerator(stubsFilePath) override val root: String? get() = System.getenv(PYCHARM_PYTHONS) } -class PyStubsGenerator : LanguageLevelAwareStubsGenerator(PyFileElementType.INSTANCE.stubVersion.toString()) { +class PyStubsGenerator(stubsStorageFilePath: String) : LanguageLevelAwareStubsGenerator(PyFileElementType.INSTANCE.stubVersion.toString(), stubsStorageFilePath) { override fun defaultLanguageLevel(): LanguageLevel = LanguageLevel.getDefault() override fun languageLevelIterator() = LanguageLevel.SUPPORTED_LEVELS.iterator() diff --git a/python/tools/src/com/jetbrains/python/tools/PyPrebuiltStubsGenerator.kt b/python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt similarity index 79% rename from python/tools/src/com/jetbrains/python/tools/PyPrebuiltStubsGenerator.kt rename to python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt index 71d15522e202..11dc26543c2d 100644 --- a/python/tools/src/com/jetbrains/python/tools/PyPrebuiltStubsGenerator.kt +++ b/python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt @@ -2,6 +2,7 @@ package com.jetbrains.python.tools import com.intellij.idea.IdeaTestApplication +import com.intellij.index.PrebuiltIndexAwareIdIndexer import com.intellij.openapi.application.PathManager import com.intellij.openapi.application.WriteAction import com.intellij.openapi.util.io.FileUtil @@ -12,21 +13,25 @@ import com.intellij.util.ReflectionUtil import com.intellij.util.io.ZipUtil import com.intellij.util.ui.UIUtil import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider +import org.jetbrains.index.id.IdIndexGenerator import java.io.File fun main(args: Array) { - buildStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") + buildIndices(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}") } -fun buildStubs(root: String, outputPath: String) { +fun buildIndices(root: String, outputPath: String) { val app = createApp() try { FileUtil.delete(File(outputPath)) val roots = unzipArchivesToRoots(root) - PyStubsGenerator().buildStubsForRoots("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}", - ArrayList(roots.map { it -> LocalFileSystem.getInstance().findFileByIoFile(it)!! })) + val rootFiles = ArrayList(roots.map { it -> LocalFileSystem.getInstance().findFileByIoFile(it)!! }) + + PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}").buildStubsForRoots(rootFiles) + + IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}").buildIdIndexForRoots(rootFiles) } catch (e: Throwable) { e.printStackTrace() diff --git a/tools/index-tools/src/org/jetbrains/index/IndexGenerator.kt b/tools/index-tools/src/org/jetbrains/index/IndexGenerator.kt new file mode 100644 index 000000000000..795ef2c23f33 --- /dev/null +++ b/tools/index-tools/src/org/jetbrains/index/IndexGenerator.kt @@ -0,0 +1,97 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.index + +import com.google.common.hash.HashCode +import com.intellij.openapi.vfs.VfsUtilCore +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.openapi.vfs.VirtualFileFilter +import com.intellij.openapi.vfs.VirtualFileVisitor +import com.intellij.psi.stubs.FileContentHashing +import com.intellij.util.indexing.FileContentImpl +import com.intellij.util.io.PersistentHashMap +import junit.framework.TestCase +import java.util.* +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author traff + */ + +abstract class IndexGenerator(private val indexStorageFilePath: String) { + companion object { + val CHECK_HASH_COLLISIONS: Boolean = System.getenv("INDEX_GENERATOR_CHECK_HASH_COLLISIONS")?.toBoolean() ?: false + } + + open val fileFilter + get() = VirtualFileFilter { f -> !f.isDirectory } + + protected fun buildIndexForRoots(roots: List) { + val hashing = FileContentHashing() + + val storage = createStorage(indexStorageFilePath) + + println("Writing indicesls to ${storage.baseFile.absolutePath}") + + try { + val map = HashMap>() + + for (file in roots) { + println("Processing files in root ${file.path}") + val cnt = AtomicInteger() + VfsUtilCore.visitChildrenRecursively(file, + object : VirtualFileVisitor() { + override fun visitFile(file: VirtualFile): Boolean { + return indexFile(file, hashing, map, storage, cnt) + } + }) + + println("${cnt.get()} entries written") + } + } + finally { + storage.close() + } + } + + private fun indexFile(file: VirtualFile, + hashing: FileContentHashing, + map: HashMap>, + storage: PersistentHashMap, + cnt: AtomicInteger): Boolean { + try { + if (fileFilter.accept(file)) { + val fileContent = FileContentImpl( + file, file.contentsToByteArray()) + + val hashCode = hashing.hashString(fileContent) + + val value = getIndexValue(fileContent) + + val item = map[hashCode] + if (item == null) { + storage.put(hashCode, value) + cnt.incrementAndGet() + + if (CHECK_HASH_COLLISIONS) { + map.put(hashCode, + Pair(fileContent.contentAsText.toString(), value)) + } + } + else { + TestCase.assertEquals(item.first, + fileContent.contentAsText.toString()) + TestCase.assertTrue(value == item.second) + } + } + } + catch (e: NoSuchElementException) { + return false + } + + return true + } + + protected abstract fun getIndexValue(fileContent: FileContentImpl): Value + + protected abstract fun createStorage(stubsStorageFilePath: String): PersistentHashMap +} \ No newline at end of file diff --git a/tools/index-tools/src/org/jetbrains/index/id/IdIndexGenerator.kt b/tools/index-tools/src/org/jetbrains/index/id/IdIndexGenerator.kt new file mode 100644 index 000000000000..159f5d43fe6f --- /dev/null +++ b/tools/index-tools/src/org/jetbrains/index/id/IdIndexGenerator.kt @@ -0,0 +1,32 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.index.id + +import com.google.common.hash.HashCode +import com.intellij.index.IdIndexMapDataExternalizer +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.psi.impl.cache.impl.id.IdIndexEntry +import com.intellij.psi.impl.cache.impl.id.IdIndexers +import com.intellij.psi.stubs.HashCodeDescriptor +import com.intellij.util.indexing.FileContentImpl +import com.intellij.util.io.PersistentHashMap +import org.jetbrains.index.IndexGenerator +import java.io.File + +/** + * @author traff + */ + +class IdIndexGenerator(private val idIndexStorageFilePath: String) : IndexGenerator>(idIndexStorageFilePath) { + override fun getIndexValue(fileContent: FileContentImpl): Map { + return IdIndexers.INSTANCE.forFileType(fileContent.file.fileType).map(fileContent) + } + + override fun createStorage(stubsStorageFilePath: String): PersistentHashMap> { + return PersistentHashMap(File(idIndexStorageFilePath + ".input"), + HashCodeDescriptor.instance, IdIndexMapDataExternalizer()) + } + + fun buildIdIndexForRoots(rootFiles: ArrayList) { + buildIndexForRoots(rootFiles) + } +} diff --git a/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt b/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt index dfaa72ff8efb..078434b24f35 100644 --- a/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt +++ b/tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt @@ -27,104 +27,62 @@ import com.intellij.openapi.project.ProjectManager import com.intellij.openapi.util.Disposer import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream import com.intellij.openapi.util.io.FileUtil -import com.intellij.openapi.vfs.VfsUtilCore import com.intellij.openapi.vfs.VirtualFile -import com.intellij.openapi.vfs.VirtualFileFilter -import com.intellij.openapi.vfs.VirtualFileVisitor import com.intellij.psi.impl.DebugUtil import com.intellij.psi.stubs.* import com.intellij.util.indexing.FileContentImpl import com.intellij.util.io.PersistentHashMap import com.intellij.util.ui.UIUtil import junit.framework.TestCase +import org.jetbrains.index.IndexGenerator import java.io.ByteArrayInputStream import java.io.File import java.util.* -import java.util.concurrent.atomic.AtomicInteger -val CHECK_COLLISIONS = false - /** * Generates stubs and stores them in one persistent hash map */ -open class StubsGenerator(private val stubsVersion: String) { +open class StubsGenerator(private val stubsVersion: String, private val stubsStorageFilePath: String) : + IndexGenerator(stubsStorageFilePath) { - open val fileFilter - get() = VirtualFileFilter { true } - - fun buildStubsForRoots(stubsStorageFilePath: String, - roots: List) { - val hashing = FileContentHashing() - - val stubExternalizer = StubTreeExternalizer() - val storage = PersistentHashMap(File(stubsStorageFilePath + ".input"), - HashCodeDescriptor.instance, stubExternalizer) - - println("Writing stubs to ${storage.baseFile.absolutePath}") - - val serializationManager = SerializationManagerImpl(File(stubsStorageFilePath + ".names")) + private val serializationManager = SerializationManagerImpl(File(stubsStorageFilePath + ".names")) + fun buildStubsForRoots(roots: List) { try { - val map = HashMap>() - - for (file in roots) { - println("Processing files in root ${file.path}") - val cnt = AtomicInteger() - VfsUtilCore.visitChildrenRecursively(file, object : VirtualFileVisitor() { - override fun visitFile(file: VirtualFile): Boolean { - try { - if (fileFilter.accept(file)) { - val fileContent = FileContentImpl(file, file.contentsToByteArray()) - val stub = buildStubForFile(fileContent, serializationManager) - val hashCode = hashing.hashString(fileContent) - - val bytes = BufferExposingByteArrayOutputStream() - serializationManager.serialize(stub, bytes) - - val contentLength = - if (file.fileType.isBinary) { - -1 - } - else { - fileContent.psiFileForPsiDependentIndex.textLength - } - - val stubTree = SerializedStubTree(bytes.internalBuffer, bytes.size(), stub, file.length, contentLength) - val item = map.get(hashCode) - if (item == null) { - storage.put(hashCode, stubTree) - cnt.incrementAndGet() - - if (CHECK_COLLISIONS) { - map.put(hashCode, Pair(fileContent.contentAsText.toString(), stubTree)) - } - } - else { - TestCase.assertEquals(item.first, fileContent.contentAsText.toString()) - TestCase.assertTrue(stubTree == item.second) - } - } - } - catch (e: NoSuchElementException) { - return false - } - - return true - } - }) - - println("${cnt.get()} entries written") - } + buildIndexForRoots(roots) } finally { - storage.close() Disposer.dispose(serializationManager) writeStubsVersionFile(stubsStorageFilePath, stubsVersion) } } + override fun getIndexValue(fileContent: FileContentImpl): SerializedStubTree { + val stub = buildStubForFile(fileContent, serializationManager) + + val bytes = BufferExposingByteArrayOutputStream() + serializationManager.serialize(stub, bytes) + + val file = fileContent.file + + val contentLength = + if (file.fileType.isBinary) { + -1 + } + else { + fileContent.psiFileForPsiDependentIndex.textLength + } + + return SerializedStubTree(bytes.internalBuffer, bytes.size(), stub, file.length, contentLength) + } + + override fun createStorage(stubsStorageFilePath: String): PersistentHashMap { + return PersistentHashMap(File(stubsStorageFilePath + ".input"), + HashCodeDescriptor.instance, StubTreeExternalizer()) + } + open fun buildStubForFile(fileContent: FileContentImpl, serializationManager: SerializationManagerImpl): Stub { @@ -250,7 +208,8 @@ fun mergeStubs(paths: List, stubsFilePath: String, stubsFileName: String * Generates stubs for file content for different language levels returned by languageLevelIterator * and checks that they are all equal. */ -abstract class LanguageLevelAwareStubsGenerator(stubsVersion: String) : StubsGenerator(stubsVersion) { +abstract class LanguageLevelAwareStubsGenerator(stubsVersion: String, stubsStorageFilePath: String) : StubsGenerator(stubsVersion, + stubsStorageFilePath) { abstract fun languageLevelIterator(): Iterator abstract fun applyLanguageLevel(level: T) diff --git a/tools/index-tools/src/org/jetbrains/index/stubs/StubsSdkGenerator.kt b/tools/index-tools/src/org/jetbrains/index/stubs/StubsSdkGenerator.kt index 1af45b8d7e00..e74eabe9e0d6 100644 --- a/tools/index-tools/src/org/jetbrains/index/stubs/StubsSdkGenerator.kt +++ b/tools/index-tools/src/org/jetbrains/index/stubs/StubsSdkGenerator.kt @@ -34,7 +34,7 @@ import java.io.File */ abstract class ProjectSdkStubsGenerator { - open fun createStubsGenerator() = StubsGenerator("") + open fun createStubsGenerator(stubsFilePath: String) = StubsGenerator("", stubsFilePath) abstract val moduleTypeId: String @@ -81,9 +81,9 @@ abstract class ProjectSdkStubsGenerator { val roots: List = sdk!!.rootProvider.getFiles(OrderRootType.CLASSES).asList() - val stubsGenerator = createStubsGenerator() + val stubsGenerator = createStubsGenerator(stubsFilePath) - stubsGenerator.buildStubsForRoots(stubsFilePath, roots) + stubsGenerator.buildStubsForRoots(roots) } finally { UIUtil.invokeAndWaitIfNeeded(Runnable {