MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Pre-build id index for Python

Browse Source
This commit is contained in:
Dmitry Trofimov
2017-10-06 21:51:44 +02:00
parent bfd94306af
commit 72014ba941
21 changed files with 441 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/FileTypeIdIndexer.java
Vendored
+4 -7
View File
@@ -16,15 +16,12 @@
package com.intellij.psi.impl.cache.impl.id;
import com.intellij.util.indexing.DataIndexer;
import com.intellij.util.indexing.FileContent;
/**
* @deprecated left for compatibility, use IdIndexer interface instead. To be removed in IntelliJ 2018.3
*
* @author Eugene Zhuravlev
* Date: Jan 16, 2008
*/
public abstract class FileTypeIdIndexer implements DataIndexer<IdIndexEntry, Integer, FileContent> {
public int getVersion() {
return 1;
}
@Deprecated
public abstract class FileTypeIdIndexer implements IdIndexer {
}
platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndex.java
Vendored
+1 -1
View File
@@ -74,7 +74,7 @@ public class IdIndex extends FileBasedIndexExtension<IdIndexEntry, Integer> {
@Override
@NotNull
public Map<IdIndexEntry, Integer> map(@NotNull final FileContent inputData) {
final FileTypeIdIndexer indexer = IdTableBuilding.getFileTypeIndexer(inputData.getFileType());
final IdIndexer indexer = IdTableBuilding.getFileTypeIndexer(inputData.getFileType());
if (indexer != null) {
return indexer.map(inputData);
}
platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexer.java
Vendored
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.psi.impl.cache.impl.id;
import com.intellij.util.indexing.DataIndexer;
import com.intellij.util.indexing.FileContent;
/**
* @author traff
*/
public interface IdIndexer extends DataIndexer<IdIndexEntry, Integer, FileContent> {
default int getVersion() {
return 1;
}
}
platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexers.java
Vendored
+1 -1
View File
@@ -21,7 +21,7 @@ import com.intellij.openapi.fileTypes.FileTypeExtension;
/**
* @author yole
*/
public class IdIndexers extends FileTypeExtension<FileTypeIdIndexer> {
public class IdIndexers extends FileTypeExtension<IdIndexer> {
public static IdIndexers INSTANCE = new IdIndexers();
private IdIndexers() {
platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java
Vendored
+5 -5
View File
@@ -47,10 +47,10 @@ public class IdTableBuilding {
void run(CharSequence chars, @Nullable char[] charsArray, int start, int end);
}
private static final Map<FileType, FileTypeIdIndexer> ourIdIndexers = new HashMap<>();
private static final Map<FileType, IdIndexer> ourIdIndexers = new HashMap<>();
@Deprecated
public static void registerIdIndexer(@NotNull FileType fileType, FileTypeIdIndexer indexer) {
public static void registerIdIndexer(@NotNull FileType fileType, IdIndexer indexer) {
ourIdIndexers.put(fileType, indexer);
}
@@ -60,14 +60,14 @@ public class IdTableBuilding {
@Nullable
public static FileTypeIdIndexer getFileTypeIndexer(FileType fileType) {
final FileTypeIdIndexer idIndexer = ourIdIndexers.get(fileType);
public static IdIndexer getFileTypeIndexer(FileType fileType) {
final IdIndexer idIndexer = ourIdIndexers.get(fileType);
if (idIndexer != null) {
return idIndexer;
}
final FileTypeIdIndexer extIndexer = IdIndexers.INSTANCE.forFileType(fileType);
final IdIndexer extIndexer = IdIndexers.INSTANCE.forFileType(fileType);
if (extIndexer != null) {
return extIndexer;
}
platform/lang-impl/src/com/intellij/index/PrebuiltIndex.kt
+113
View File
@@ -0,0 +1,113 @@
// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.index
import com.google.common.hash.HashCode
import com.intellij.openapi.Disposable
import com.intellij.openapi.application.PathManager
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.util.io.FileUtil
import com.intellij.psi.stubs.FileContentHashing
import com.intellij.psi.stubs.HashCodeDescriptor
import com.intellij.psi.stubs.PrebuiltStubsProviderBase
import com.intellij.util.indexing.FileContent
import com.intellij.util.indexing.IndexInfrastructure
import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.PersistentHashMap
import java.io.File
import java.io.FileFilter
import java.io.IOException
/**
* @author traff
*/
abstract class PrebuiltIndexProviderBase<Value> : Disposable {
private val myFileContentHashing = FileContentHashing()
private var myPrebuiltIndexStorage: PersistentHashMap<HashCode, Value>? = null
protected abstract val dirName: String
protected abstract val indexName: String
protected abstract val indexExternalizer: DataExternalizer<Value>
companion object {
private val LOG = Logger.getInstance("#com.intellij.index.PrebuiltIndexProviderBase")
}
init {
init()
}
internal fun init() {
var indexesRoot = findPrebuiltIndicesRoot()
try {
if (indexesRoot != null) {
// we should copy prebuilt indexes to a writable folder
indexesRoot = copyPrebuiltIndicesToIndexRoot(indexesRoot)
// otherwise we can get access denied error, because persistent hash map opens file for read and write
myPrebuiltIndexStorage = openIndexStorage(indexesRoot)
LOG.info("Using prebuilt $indexName from " + myPrebuiltIndexStorage!!.baseFile.absolutePath)
}
}
catch (e: Exception) {
myPrebuiltIndexStorage = null
LOG.warn("Prebuilt indices can't be loaded at " + indexesRoot!!, e)
}
}
fun get(fileContent: FileContent): Value? {
if (myPrebuiltIndexStorage != null) {
val hashCode = myFileContentHashing.hashString(fileContent)
try {
return myPrebuiltIndexStorage!!.get(hashCode)
}
catch (e: Exception) {
LOG.error("Error reading prebuilt stubs from " + myPrebuiltIndexStorage!!.baseFile.path, e)
myPrebuiltIndexStorage = null
}
}
return null
}
open fun openIndexStorage(indexesRoot: File): PersistentHashMap<HashCode, Value>? {
return object : PersistentHashMap<HashCode, Value>(
File(indexesRoot, indexName + ".input"),
HashCodeDescriptor.instance,
indexExternalizer) {
override fun isReadOnly(): Boolean {
return true
}
}
}
@Throws(IOException::class)
private fun copyPrebuiltIndicesToIndexRoot(prebuiltIndicesRoot: File): File {
val indexRoot = File(IndexInfrastructure.getPersistentIndexRoot(), "prebuilt/" + dirName)
FileUtil.copyDir(prebuiltIndicesRoot, indexRoot, FileFilter { f -> f.nameWithoutExtension == indexName })
return indexRoot
}
private fun findPrebuiltIndicesRoot(): File? {
val path: String? = System.getProperty(PrebuiltStubsProviderBase.PREBUILT_INDICES_PATH_PROPERTY)
if (path != null && File(path).exists()) {
return File(path)
}
val f = indexRoot()
return if (f.exists()) f else null
}
open fun indexRoot(): File = File(PathManager.getHomePath(), "index/$dirName") // compiled binary
override fun dispose() {
if (myPrebuiltIndexStorage != null) {
try {
myPrebuiltIndexStorage!!.close()
}
catch (e: IOException) {
LOG.error(e)
}
}
}
}
platform/lang-impl/src/com/intellij/index/PrebuiltIndexAwareIdIndexer.kt
+57
View File
@@ -0,0 +1,57 @@
// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.index
import com.intellij.psi.impl.cache.impl.id.IdIndexEntry
import com.intellij.psi.impl.cache.impl.id.IdIndexer
import com.intellij.util.indexing.FileContent
import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.DataInputOutputUtil
import java.io.DataInput
import java.io.DataOutput
/**
* @author traff
*/
abstract class PrebuiltIndexAwareIdIndexer : PrebuiltIndexProviderBase<Map<IdIndexEntry, Int>>(), IdIndexer {
companion object {
val ID_INDEX_FILE_NAME = "id-index"
}
override val indexName = ID_INDEX_FILE_NAME
override val indexExternalizer = IdIndexMapDataExternalizer()
override fun map(inputData: FileContent): Map<IdIndexEntry, Int> {
val map = get(inputData)
return if (map != null) {
map
}
else {
idIndexMap(inputData)
}
}
abstract fun idIndexMap(inputData: FileContent): Map<IdIndexEntry, Int>
}
class IdIndexMapDataExternalizer : DataExternalizer<Map<IdIndexEntry, Int>> {
override fun save(out: DataOutput, value: Map<IdIndexEntry, Int>) {
DataInputOutputUtil.writeINT(out, value.size)
for (e in value.entries) {
DataInputOutputUtil.writeINT(out, e.key.wordHashCode)
DataInputOutputUtil.writeINT(out, e.value)
}
}
override fun read(`in`: DataInput): Map<IdIndexEntry, Int> {
val size = DataInputOutputUtil.readINT(`in`)
val map = HashMap<IdIndexEntry, Int>()
for (i in 0..size) {
val wordHash = DataInputOutputUtil.readINT(`in`)
val value= DataInputOutputUtil.readINT(`in`)
map.put(IdIndexEntry(wordHash), value)
}
return map
}
}
platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdIndexImpl.java
Vendored
+1 -2
View File
@@ -30,7 +30,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
public class IdIndexImpl extends IdIndex implements CustomInputsIndexFileBasedIndexExtension<IdIndexEntry> {
private static final ThreadLocalCachedIntArray spareBufferLocal = new ThreadLocalCachedIntArray();
@@ -48,7 +47,7 @@ public class IdIndexImpl extends IdIndex implements CustomInputsIndexFileBasedIn
int version = super.getVersion();
for(FileType fileType:types) {
if (!isIndexable(fileType)) continue;
FileTypeIdIndexer indexer = IdTableBuilding.getFileTypeIndexer(fileType);
IdIndexer indexer = IdTableBuilding.getFileTypeIndexer(fileType);
if (indexer == null) continue;
version = version * 31 + (indexer.getVersion() ^ indexer.getClass().getName().hashCode());
}
platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt
+27 -93
View File
@@ -17,15 +17,13 @@ package com.intellij.psi.stubs
import com.google.common.hash.HashCode
import com.google.common.hash.Hashing
import com.intellij.openapi.Disposable
import com.intellij.index.PrebuiltIndexProviderBase
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.application.PathManager
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.fileTypes.FileTypeExtension
import com.intellij.openapi.util.Disposer
import com.intellij.openapi.util.io.FileUtil
import com.intellij.util.indexing.FileContent
import com.intellij.util.indexing.IndexInfrastructure
import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.DataInputOutputUtil
import com.intellij.util.io.KeyDescriptor
@@ -35,7 +33,6 @@ import org.jetbrains.annotations.TestOnly
import java.io.DataInput
import java.io.DataOutput
import java.io.File
import java.io.IOException
/**
* @author traff
@@ -53,7 +50,7 @@ interface PrebuiltStubsProvider {
class FileContentHashing {
private val hashing = Hashing.sha256()
fun hashString(fileContent: FileContent) = hashing.hashBytes(fileContent.content)
fun hashString(fileContent: FileContent) = hashing.hashBytes(fileContent.content)!!
}
@@ -90,17 +87,15 @@ class StubTreeExternalizer : DataExternalizer<SerializedStubTree> {
override fun read(`in`: DataInput) = SerializedStubTree(`in`)
}
abstract class PrebuiltStubsProviderBase : PrebuiltStubsProvider, Disposable {
private val myFileContentHashing = FileContentHashing()
private var myPrebuiltStubsStorage: PersistentHashMap<HashCode, SerializedStubTree>? = null
abstract class PrebuiltStubsProviderBase : PrebuiltIndexProviderBase<SerializedStubTree>(), PrebuiltStubsProvider {
private var mySerializationManager: SerializationManagerImpl? = null
protected abstract val stubVersion: Int
protected abstract val name: String
init {
init()
}
override val indexName = SDK_STUBS_STORAGE_NAME
override val indexExternalizer = StubTreeExternalizer()
companion object {
val PREBUILT_INDICES_PATH_PROPERTY = "prebuilt_indices_path"
@@ -108,100 +103,39 @@ abstract class PrebuiltStubsProviderBase : PrebuiltStubsProvider, Disposable {
private val LOG = Logger.getInstance("#com.intellij.psi.stubs.PrebuiltStubsProviderBase")
}
internal fun init() {
var indexesRoot = findPrebuiltIndicesRoot()
try {
if (indexesRoot != null) {
// we should copy prebuilt indexes to a writable folder
indexesRoot = copyPrebuiltIndicesToIndexRoot(indexesRoot)
// otherwise we can get access denied error, because persistent hash map opens file for read and write
override fun openIndexStorage(indexesRoot: File): PersistentHashMap<HashCode, SerializedStubTree>? {
val versionInFile = FileUtil.loadFile(File(indexesRoot, indexName + ".version"))
val versionInFile = FileUtil.loadFile(File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".version"))
if (Integer.parseInt(versionInFile) == stubVersion) {
mySerializationManager = SerializationManagerImpl(File(indexesRoot, indexName + ".names"))
if (Integer.parseInt(versionInFile) == stubVersion) {
myPrebuiltStubsStorage = object : PersistentHashMap<HashCode, SerializedStubTree>(
File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".input"),
HashCodeDescriptor.instance,
StubTreeExternalizer()) {
override fun isReadOnly(): Boolean {
return true
}
}
Disposer.register(ApplicationManager.getApplication(), mySerializationManager!!)
mySerializationManager = SerializationManagerImpl(File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".names"))
Disposer.register(ApplicationManager.getApplication(), mySerializationManager!!)
LOG.info("Using prebuilt stubs from " + myPrebuiltStubsStorage!!.baseFile.absolutePath)
}
else {
LOG.error("Prebuilt stubs version mismatch: $versionInFile, current version is $stubVersion")
}
}
return super.openIndexStorage(indexesRoot)
}
catch (e: Exception) {
myPrebuiltStubsStorage = null
LOG.warn("Prebuilt stubs can't be loaded at " + indexesRoot!!, e)
else {
LOG.error("Prebuilt stubs version mismatch: $versionInFile, current version is $stubVersion")
return null
}
}
override fun findStub(fileContent: FileContent): Stub? {
if (myPrebuiltStubsStorage != null) {
val hashCode = myFileContentHashing.hashString(fileContent)
var stub: Stub? = null
try {
val stubTree = myPrebuiltStubsStorage!!.get(hashCode)
if (stubTree != null) {
stub = stubTree.getStub(false, mySerializationManager!!)
}
}
catch (e: SerializerNotFoundException) {
LOG.error("Can't deserialize stub tree", e)
}
catch (e: Exception) {
LOG.error("Error reading prebuilt stubs from " + myPrebuiltStubsStorage!!.baseFile.path, e)
myPrebuiltStubsStorage = null
stub = null
var stub: Stub? = null
try {
val stubTree = get(fileContent)
if (stubTree != null) {
stub = stubTree.getStub(false, mySerializationManager!!)
}
}
catch (e: SerializerNotFoundException) {
LOG.error("Can't deserialize stub tree", e)
}
if (stub != null) {
return stub
}
if (stub != null) {
return stub
}
return null
}
override fun dispose() {
if (myPrebuiltStubsStorage != null) {
try {
myPrebuiltStubsStorage!!.close()
}
catch (e: IOException) {
LOG.error(e)
}
}
}
@Throws(IOException::class)
private fun copyPrebuiltIndicesToIndexRoot(prebuiltIndicesRoot: File): File {
val indexRoot = File(IndexInfrastructure.getPersistentIndexRoot(), "prebuilt/" + name)
FileUtil.copyDir(prebuiltIndicesRoot, indexRoot)
return indexRoot
}
private fun findPrebuiltIndicesRoot(): File? {
val path: String? = System.getProperty(PREBUILT_INDICES_PATH_PROPERTY)
if (path != null && File(path).exists()) {
return File(path)
}
val f = indexRoot()
return if (f.exists()) f else null
}
open fun indexRoot():File = File(PathManager.getHomePath(), "index/$name") // compiled binary
}
@TestOnly
platform/platform-resources/src/META-INF/LangExtensionPoints.xml
+1 -1
View File
@@ -780,7 +780,7 @@
<extensionPoint name="fileLookupInfoProvider" interface="com.intellij.psi.file.FileLookupInfoProvider"/>
<extensionPoint name="idIndexer" beanClass="com.intellij.openapi.fileTypes.FileTypeExtensionPoint">
<with attribute="implementationClass" implements="com.intellij.psi.impl.cache.impl.id.FileTypeIdIndexer"/>
<with attribute="implementationClass" implements="com.intellij.psi.impl.cache.impl.id.IdIndexer"/>
</extensionPoint>
<extensionPoint name="todoIndexer" beanClass="com.intellij.openapi.fileTypes.FileTypeExtensionPoint">
python/build/groovy/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.groovy
+4 -4
View File
@@ -43,7 +43,7 @@ abstract class PyCharmPropertiesBase extends ProductProperties {
}
}
new PyPrebuiltStubsGenerator().generateResources(context)
new PyPrebuiltIndicesGenerator().generateResources(context)
context.ant.copy(todir: "$targetDirectory/index", failonerror: true) {
fileset(dir: "$context.paths.temp/index", erroronmissingdir: true) {
@@ -58,7 +58,7 @@ abstract class PyCharmPropertiesBase extends ProductProperties {
}
}
class PyPrebuiltStubsGenerator implements ResourcesGenerator {
class PyPrebuiltIndicesGenerator implements ResourcesGenerator {
@Override
File generateResources(BuildContext context) {
CompilationTasks.create(context).compileModules(["python-community-tools"])
@@ -77,8 +77,8 @@ class PyPrebuiltStubsGenerator implements ResourcesGenerator {
def outputPath = "$context.paths.temp/index"
context.ant.java(classname: "com.jetbrains.python.tools.PyPrebuiltStubsGeneratorKt", fork: true) {
jvmarg(line: "-ea -Xmx1500m")
context.ant.java(classname: "com.jetbrains.python.tools.PyPrebuiltIndicesGeneratorKt", fork: true) {
jvmarg(line: "-ea -Xmx1000m")
arg(value: zipPath)
arg(value: outputPath)
classpath {
python/src/com/jetbrains/python/PyIdIndexer.java
-35
View File
@@ -1,35 +0,0 @@
/*
* Copyright 2000-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python;
import com.intellij.lexer.Lexer;
import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer;
import com.jetbrains.python.lexer.PythonLexer;
/**
* @author traff
*/
public class PyIdIndexer extends LexerBasedIdIndexer {
@Override
public Lexer createLexer(OccurrenceConsumer consumer) {
return createIndexingLexer(consumer);
}
public static Lexer createIndexingLexer(OccurrenceConsumer consumer) {
return new PyFilterLexer(new PythonLexer(), consumer);
}
}
python/src/com/jetbrains/python/PyIdIndexer.kt
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python
import com.intellij.index.PrebuiltIndexAwareIdIndexer
import com.intellij.lexer.Lexer
import com.intellij.psi.impl.cache.impl.OccurrenceConsumer
import com.intellij.psi.impl.cache.impl.id.IdIndexEntry
import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer
import com.intellij.util.indexing.FileContent
import com.jetbrains.python.lexer.PythonLexer
import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider
/**
* @author traff
*/
class PyIdIndexer : PrebuiltIndexAwareIdIndexer() {
private val myIndexer = MyPyIdIndexer()
override val dirName: String = PyPrebuiltStubsProvider.NAME
override fun idIndexMap(inputData: FileContent): Map<IdIndexEntry, Int> {
return myIndexer.map(inputData)
}
companion object {
fun createIndexingLexer(consumer: OccurrenceConsumer): Lexer {
return PyFilterLexer(PythonLexer(), consumer)
}
}
private class MyPyIdIndexer : LexerBasedIdIndexer() {
override fun createLexer(consumer: OccurrenceConsumer): Lexer {
return createIndexingLexer(consumer)
}
}
}
python/src/com/jetbrains/python/PyTodoIndexer.java
+1 -1
View File
@@ -25,6 +25,6 @@ import com.intellij.psi.impl.cache.impl.todo.LexerBasedTodoIndexer;
public class PyTodoIndexer extends LexerBasedTodoIndexer {
@Override
public Lexer createLexer(OccurrenceConsumer consumer) {
return PyIdIndexer.createIndexingLexer(consumer);
return PyIdIndexer.Companion.createIndexingLexer(consumer);
}
}
python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java
+1 -1
View File
@@ -34,7 +34,7 @@ public class PyPrebuiltStubsProvider extends PrebuiltStubsProviderBase {
@NotNull
@Override
protected String getName() {
protected String getDirName() {
return NAME;
}
}
python/tools/src/com/jetbrains/python/tools/BuildStubsForSdk.kt
+2 -2
View File
@@ -90,13 +90,13 @@ class PyProjectSdkStubsGenerator : ProjectSdkStubsGenerator() {
override fun createSdkProducer(sdkPath: String) = createPythonSdkProducer(sdkPath)
override fun createStubsGenerator() = PyStubsGenerator()
override fun createStubsGenerator(stubsFilePath: String) = PyStubsGenerator(stubsFilePath)
override val root: String?
get() = System.getenv(PYCHARM_PYTHONS)
}
class PyStubsGenerator : LanguageLevelAwareStubsGenerator<LanguageLevel>(PyFileElementType.INSTANCE.stubVersion.toString()) {
class PyStubsGenerator(stubsStorageFilePath: String) : LanguageLevelAwareStubsGenerator<LanguageLevel>(PyFileElementType.INSTANCE.stubVersion.toString(), stubsStorageFilePath) {
override fun defaultLanguageLevel(): LanguageLevel = LanguageLevel.getDefault()
override fun languageLevelIterator() = LanguageLevel.SUPPORTED_LEVELS.iterator()
python/tools/src/com/jetbrains/python/tools/PyPrebuiltStubsGenerator.kt → python/tools/src/com/jetbrains/python/tools/PyPrebuiltIndicesGenerator.kt
+9 -4
View File
@@ -2,6 +2,7 @@
package com.jetbrains.python.tools
import com.intellij.idea.IdeaTestApplication
import com.intellij.index.PrebuiltIndexAwareIdIndexer
import com.intellij.openapi.application.PathManager
import com.intellij.openapi.application.WriteAction
import com.intellij.openapi.util.io.FileUtil
@@ -12,21 +13,25 @@ import com.intellij.util.ReflectionUtil
import com.intellij.util.io.ZipUtil
import com.intellij.util.ui.UIUtil
import com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider
import org.jetbrains.index.id.IdIndexGenerator
import java.io.File
fun main(args: Array<String>) {
buildStubs(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
buildIndices(args[0], "${args[1]}/${PyPrebuiltStubsProvider.NAME}")
}
fun buildStubs(root: String, outputPath: String) {
fun buildIndices(root: String, outputPath: String) {
val app = createApp()
try {
FileUtil.delete(File(outputPath))
val roots = unzipArchivesToRoots(root)
PyStubsGenerator().buildStubsForRoots("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}",
ArrayList(roots.map { it -> LocalFileSystem.getInstance().findFileByIoFile(it)!! }))
val rootFiles = ArrayList(roots.map { it -> LocalFileSystem.getInstance().findFileByIoFile(it)!! })
PyStubsGenerator("$outputPath/${PrebuiltStubsProviderBase.SDK_STUBS_STORAGE_NAME}").buildStubsForRoots(rootFiles)
IdIndexGenerator("$outputPath/${PrebuiltIndexAwareIdIndexer.ID_INDEX_FILE_NAME}").buildIdIndexForRoots(rootFiles)
}
catch (e: Throwable) {
e.printStackTrace()
tools/index-tools/src/org/jetbrains/index/IndexGenerator.kt
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.index
import com.google.common.hash.HashCode
import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.VirtualFileFilter
import com.intellij.openapi.vfs.VirtualFileVisitor
import com.intellij.psi.stubs.FileContentHashing
import com.intellij.util.indexing.FileContentImpl
import com.intellij.util.io.PersistentHashMap
import junit.framework.TestCase
import java.util.*
import java.util.concurrent.atomic.AtomicInteger
/**
* @author traff
*/
abstract class IndexGenerator<Value>(private val indexStorageFilePath: String) {
companion object {
val CHECK_HASH_COLLISIONS: Boolean = System.getenv("INDEX_GENERATOR_CHECK_HASH_COLLISIONS")?.toBoolean() ?: false
}
open val fileFilter
get() = VirtualFileFilter { f -> !f.isDirectory }
protected fun buildIndexForRoots(roots: List<VirtualFile>) {
val hashing = FileContentHashing()
val storage = createStorage(indexStorageFilePath)
println("Writing indicesls to ${storage.baseFile.absolutePath}")
try {
val map = HashMap<HashCode, Pair<String, Value>>()
for (file in roots) {
println("Processing files in root ${file.path}")
val cnt = AtomicInteger()
VfsUtilCore.visitChildrenRecursively(file,
object : VirtualFileVisitor<Boolean>() {
override fun visitFile(file: VirtualFile): Boolean {
return indexFile(file, hashing, map, storage, cnt)
}
})
println("${cnt.get()} entries written")
}
}
finally {
storage.close()
}
}
private fun indexFile(file: VirtualFile,
hashing: FileContentHashing,
map: HashMap<HashCode, Pair<String, Value>>,
storage: PersistentHashMap<HashCode, Value>,
cnt: AtomicInteger): Boolean {
try {
if (fileFilter.accept(file)) {
val fileContent = FileContentImpl(
file, file.contentsToByteArray())
val hashCode = hashing.hashString(fileContent)
val value = getIndexValue(fileContent)
val item = map[hashCode]
if (item == null) {
storage.put(hashCode, value)
cnt.incrementAndGet()
if (CHECK_HASH_COLLISIONS) {
map.put(hashCode,
Pair(fileContent.contentAsText.toString(), value))
}
}
else {
TestCase.assertEquals(item.first,
fileContent.contentAsText.toString())
TestCase.assertTrue(value == item.second)
}
}
}
catch (e: NoSuchElementException) {
return false
}
return true
}
protected abstract fun getIndexValue(fileContent: FileContentImpl): Value
protected abstract fun createStorage(stubsStorageFilePath: String): PersistentHashMap<HashCode, Value>
}
tools/index-tools/src/org/jetbrains/index/id/IdIndexGenerator.kt
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.index.id
import com.google.common.hash.HashCode
import com.intellij.index.IdIndexMapDataExternalizer
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.impl.cache.impl.id.IdIndexEntry
import com.intellij.psi.impl.cache.impl.id.IdIndexers
import com.intellij.psi.stubs.HashCodeDescriptor
import com.intellij.util.indexing.FileContentImpl
import com.intellij.util.io.PersistentHashMap
import org.jetbrains.index.IndexGenerator
import java.io.File
/**
* @author traff
*/
class IdIndexGenerator(private val idIndexStorageFilePath: String) : IndexGenerator<Map<IdIndexEntry, Int>>(idIndexStorageFilePath) {
override fun getIndexValue(fileContent: FileContentImpl): Map<IdIndexEntry, Int> {
return IdIndexers.INSTANCE.forFileType(fileContent.file.fileType).map(fileContent)
}
override fun createStorage(stubsStorageFilePath: String): PersistentHashMap<HashCode, Map<IdIndexEntry, Int>> {
return PersistentHashMap(File(idIndexStorageFilePath + ".input"),
HashCodeDescriptor.instance, IdIndexMapDataExternalizer())
}
fun buildIdIndexForRoots(rootFiles: ArrayList<VirtualFile>) {
buildIndexForRoots(rootFiles)
}
}
tools/index-tools/src/org/jetbrains/index/stubs/StubsGenerator.kt
+32 -73
View File
@@ -27,104 +27,62 @@ import com.intellij.openapi.project.ProjectManager
import com.intellij.openapi.util.Disposer
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.VirtualFileFilter
import com.intellij.openapi.vfs.VirtualFileVisitor
import com.intellij.psi.impl.DebugUtil
import com.intellij.psi.stubs.*
import com.intellij.util.indexing.FileContentImpl
import com.intellij.util.io.PersistentHashMap
import com.intellij.util.ui.UIUtil
import junit.framework.TestCase
import org.jetbrains.index.IndexGenerator
import java.io.ByteArrayInputStream
import java.io.File
import java.util.*
import java.util.concurrent.atomic.AtomicInteger
val CHECK_COLLISIONS = false
/**
* Generates stubs and stores them in one persistent hash map
*/
open class StubsGenerator(private val stubsVersion: String) {
open class StubsGenerator(private val stubsVersion: String, private val stubsStorageFilePath: String) :
IndexGenerator<SerializedStubTree>(stubsStorageFilePath) {
open val fileFilter
get() = VirtualFileFilter { true }
fun buildStubsForRoots(stubsStorageFilePath: String,
roots: List<VirtualFile>) {
val hashing = FileContentHashing()
val stubExternalizer = StubTreeExternalizer()
val storage = PersistentHashMap<HashCode, SerializedStubTree>(File(stubsStorageFilePath + ".input"),
HashCodeDescriptor.instance, stubExternalizer)
println("Writing stubs to ${storage.baseFile.absolutePath}")
val serializationManager = SerializationManagerImpl(File(stubsStorageFilePath + ".names"))
private val serializationManager = SerializationManagerImpl(File(stubsStorageFilePath + ".names"))
fun buildStubsForRoots(roots: List<VirtualFile>) {
try {
val map = HashMap<HashCode, Pair<String, SerializedStubTree>>()
for (file in roots) {
println("Processing files in root ${file.path}")
val cnt = AtomicInteger()
VfsUtilCore.visitChildrenRecursively(file, object : VirtualFileVisitor<Boolean>() {
override fun visitFile(file: VirtualFile): Boolean {
try {
if (fileFilter.accept(file)) {
val fileContent = FileContentImpl(file, file.contentsToByteArray())
val stub = buildStubForFile(fileContent, serializationManager)
val hashCode = hashing.hashString(fileContent)
val bytes = BufferExposingByteArrayOutputStream()
serializationManager.serialize(stub, bytes)
val contentLength =
if (file.fileType.isBinary) {
-1
}
else {
fileContent.psiFileForPsiDependentIndex.textLength
}
val stubTree = SerializedStubTree(bytes.internalBuffer, bytes.size(), stub, file.length, contentLength)
val item = map.get(hashCode)
if (item == null) {
storage.put(hashCode, stubTree)
cnt.incrementAndGet()
if (CHECK_COLLISIONS) {
map.put(hashCode, Pair(fileContent.contentAsText.toString(), stubTree))
}
}
else {
TestCase.assertEquals(item.first, fileContent.contentAsText.toString())
TestCase.assertTrue(stubTree == item.second)
}
}
}
catch (e: NoSuchElementException) {
return false
}
return true
}
})
println("${cnt.get()} entries written")
}
buildIndexForRoots(roots)
}
finally {
storage.close()
Disposer.dispose(serializationManager)
writeStubsVersionFile(stubsStorageFilePath, stubsVersion)
}
}
override fun getIndexValue(fileContent: FileContentImpl): SerializedStubTree {
val stub = buildStubForFile(fileContent, serializationManager)
val bytes = BufferExposingByteArrayOutputStream()
serializationManager.serialize(stub, bytes)
val file = fileContent.file
val contentLength =
if (file.fileType.isBinary) {
-1
}
else {
fileContent.psiFileForPsiDependentIndex.textLength
}
return SerializedStubTree(bytes.internalBuffer, bytes.size(), stub, file.length, contentLength)
}
override fun createStorage(stubsStorageFilePath: String): PersistentHashMap<HashCode, SerializedStubTree> {
return PersistentHashMap(File(stubsStorageFilePath + ".input"),
HashCodeDescriptor.instance, StubTreeExternalizer())
}
open fun buildStubForFile(fileContent: FileContentImpl,
serializationManager: SerializationManagerImpl): Stub {
@@ -250,7 +208,8 @@ fun mergeStubs(paths: List<String>, stubsFilePath: String, stubsFileName: String
* Generates stubs for file content for different language levels returned by languageLevelIterator
* and checks that they are all equal.
*/
abstract class LanguageLevelAwareStubsGenerator<T>(stubsVersion: String) : StubsGenerator(stubsVersion) {
abstract class LanguageLevelAwareStubsGenerator<T>(stubsVersion: String, stubsStorageFilePath: String) : StubsGenerator(stubsVersion,
stubsStorageFilePath) {
abstract fun languageLevelIterator(): Iterator<T>
abstract fun applyLanguageLevel(level: T)
tools/index-tools/src/org/jetbrains/index/stubs/StubsSdkGenerator.kt
+3 -3
View File
@@ -34,7 +34,7 @@ import java.io.File
*/
abstract class ProjectSdkStubsGenerator {
open fun createStubsGenerator() = StubsGenerator("")
open fun createStubsGenerator(stubsFilePath: String) = StubsGenerator("", stubsFilePath)
abstract val moduleTypeId: String
@@ -81,9 +81,9 @@ abstract class ProjectSdkStubsGenerator {
val roots: List<VirtualFile> = sdk!!.rootProvider.getFiles(OrderRootType.CLASSES).asList()
val stubsGenerator = createStubsGenerator()
val stubsGenerator = createStubsGenerator(stubsFilePath)
stubsGenerator.buildStubsForRoots(stubsFilePath, roots)
stubsGenerator.buildStubsForRoots(roots)
}
finally {
UIUtil.invokeAndWaitIfNeeded(Runnable {