From e34f3578be21f00385d4be96dbc2482448e9f7fb Mon Sep 17 00:00:00 2001 From: Max Medvedev Date: Mon, 30 Sep 2024 22:58:54 +0200 Subject: [PATCH] IJPL-149042 fix INRE on building java classes it's not safe to use PsiClass#getMethods during building indexes GitOrigin-RevId: a7828cf93e8004b6cd55f743a7ff0958a53afcbb --- .../ml/embeddings/indexer/ClassesProvider.kt | 8 +++--- .../ml/embeddings/indexer/SymbolsProvider.kt | 8 +++--- .../ml/embeddings/indexer/embeddingIndexes.kt | 8 +++--- .../vfs/VFSBasedEmbeddingEntitiesIndexer.kt | 16 +++++++++--- ...llij.searchEverywhereMl.semantics.java.iml | 2 +- .../semantics/java/JavaClassesProvider.kt | 23 +++++++++++----- .../semantics/java/JavaSymbolsProvider.kt | 26 ++++++++++++++----- .../semantics/kotlin/KotlinClassesProvider.kt | 6 ++--- .../semantics/kotlin/KotlinSymbolsProvider.kt | 6 ++--- .../PythonClassesProvider.kt | 6 ++--- .../PythonSymbolsProvider.kt | 6 ++--- 11 files changed, 74 insertions(+), 41 deletions(-) diff --git a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/ClassesProvider.kt b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/ClassesProvider.kt index d011989990d3..6c77bee1fa6e 100644 --- a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/ClassesProvider.kt +++ b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/ClassesProvider.kt @@ -4,15 +4,15 @@ package com.intellij.platform.ml.embeddings.indexer import com.intellij.openapi.fileTypes.FileType import com.intellij.openapi.fileTypes.FileTypeExtension import com.intellij.platform.ml.embeddings.indexer.entities.IndexableClass -import com.intellij.psi.PsiFile import com.intellij.util.concurrency.ThreadingAssertions import com.intellij.util.concurrency.annotations.RequiresReadLock import com.intellij.util.containers.toArray +import com.intellij.util.indexing.FileContent import org.jetbrains.annotations.ApiStatus.Internal @Internal interface ClassesProvider { - fun extract(file: PsiFile): List + fun extract(fileContent: FileContent): List companion object { private val EXTENSION = FileTypeExtension("com.intellij.embeddings.indexer.classesProvider") @@ -21,9 +21,9 @@ interface ClassesProvider { get() = EXTENSION.getAllRegisteredExtensions().keys.toArray(FileType.EMPTY_ARRAY) @RequiresReadLock - fun extractClasses(file: PsiFile): List { + fun extractClasses(fileContent: FileContent): List { ThreadingAssertions.assertReadAccess() // annotation doesn't work in Kotlin - return EXTENSION.forFileType(file.fileType)?.extract(file) ?: emptyList() + return EXTENSION.forFileType(fileContent.fileType)?.extract(fileContent) ?: emptyList() } } } \ No newline at end of file diff --git a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/SymbolsProvider.kt b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/SymbolsProvider.kt index ae7533684d5c..67ec8261e904 100644 --- a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/SymbolsProvider.kt +++ b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/SymbolsProvider.kt @@ -4,15 +4,15 @@ package com.intellij.platform.ml.embeddings.indexer import com.intellij.openapi.fileTypes.FileType import com.intellij.openapi.fileTypes.FileTypeExtension import com.intellij.platform.ml.embeddings.indexer.entities.IndexableSymbol -import com.intellij.psi.PsiFile import com.intellij.util.concurrency.ThreadingAssertions import com.intellij.util.concurrency.annotations.RequiresReadLock import com.intellij.util.containers.toArray +import com.intellij.util.indexing.FileContent import org.jetbrains.annotations.ApiStatus.Internal @Internal interface SymbolsProvider { - fun extract(file: PsiFile): List + fun extract(fileContent: FileContent): List companion object { private val EXTENSION = FileTypeExtension("com.intellij.embeddings.indexer.symbolsProvider") @@ -21,9 +21,9 @@ interface SymbolsProvider { get() = EXTENSION.getAllRegisteredExtensions().keys.toArray(FileType.EMPTY_ARRAY) @RequiresReadLock - fun extractSymbols(file: PsiFile): List { + fun extractSymbols(fileContent: FileContent): List { ThreadingAssertions.assertReadAccess() // annotation doesn't work in Kotlin - return EXTENSION.forFileType(file.fileType)?.extract(file) ?: emptyList() + return EXTENSION.forFileType(fileContent.fileType)?.extract(fileContent) ?: emptyList() } } } \ No newline at end of file diff --git a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/embeddingIndexes.kt b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/embeddingIndexes.kt index ad6c78d0bbba..9c429c692529 100644 --- a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/embeddingIndexes.kt +++ b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/embeddingIndexes.kt @@ -50,9 +50,9 @@ internal class ClassNameEmbeddingIndex : PsiBaseEmbeddingIndex() { get() = ClassesProvider.supportedFileTypes override fun getName(): ID = CLASS_NAME_EMBEDDING_INDEX_NAME - override fun getVersion(): Int = 1 + override fun getVersion(): Int = 2 override fun index(inputData: FileContent): List { - return ClassesProvider.extractClasses(inputData.psiFile).map { IndexingItem(it.id.id) } + return ClassesProvider.extractClasses(inputData).map { IndexingItem(it.id.id) } } } @@ -61,9 +61,9 @@ internal class SymbolNameEmbeddingIndex : PsiBaseEmbeddingIndex() { get() = SymbolsProvider.supportedFileTypes override fun getName(): ID = SYMBOL_NAME_EMBEDDING_INDEX_NAME - override fun getVersion(): Int = 1 + override fun getVersion(): Int = 2 override fun index(inputData: FileContent): List { - return SymbolsProvider.extractSymbols(inputData.psiFile).map { IndexingItem(it.id.id) } + return SymbolsProvider.extractSymbols(inputData).map { IndexingItem(it.id.id) } } } diff --git a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/searcher/vfs/VFSBasedEmbeddingEntitiesIndexer.kt b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/searcher/vfs/VFSBasedEmbeddingEntitiesIndexer.kt index 86e397accf8c..50775cca474d 100644 --- a/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/searcher/vfs/VFSBasedEmbeddingEntitiesIndexer.kt +++ b/platform/ml-embeddings/src/com/intellij/platform/ml/embeddings/indexer/searcher/vfs/VFSBasedEmbeddingEntitiesIndexer.kt @@ -26,6 +26,8 @@ import com.intellij.platform.ml.embeddings.settings.EmbeddingIndexSettingsImpl import com.intellij.platform.ml.embeddings.utils.SEMANTIC_SEARCH_TRACER import com.intellij.platform.util.coroutines.childScope import com.intellij.psi.PsiManager +import com.intellij.util.indexing.FileContentImpl +import com.intellij.util.indexing.IndexingDataKeys.PSI_FILE import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import java.util.concurrent.atomic.AtomicBoolean @@ -149,16 +151,24 @@ internal class VFSBasedEmbeddingEntitiesIndexer(private val cs: CoroutineScope) } if (classesChannel != null || symbolsChannel != null) { - val psiFile = readActionUndispatched { psiManager.findFile(file) } ?: return@coroutineScope + val (psiFile, fileType) = readActionUndispatched { + val psiFile = psiManager.findFile(file) ?: return@readActionUndispatched null + val fileType = psiFile.fileType + psiFile to fileType + } ?: return@coroutineScope + + val content: FileContentImpl = FileContentImpl.createByFile(file, psiManager.project) as FileContentImpl + content.putUserData(PSI_FILE, psiFile) // todo I think we can avoid explicit passing of file + content.setSubstituteFileType(fileType) // todo we don't want to infer the substituted file type, do we? if (classesChannel != null) { launch { - readActionUndispatched { ClassesProvider.extractClasses(psiFile) }.forEach { classesChannel.send(it) } + readActionUndispatched { ClassesProvider.extractClasses(content) }.forEach { classesChannel.send(it) } } } if (symbolsChannel != null) { launch { - readActionUndispatched { SymbolsProvider.extractSymbols(psiFile) }.forEach { symbolsChannel.send(it) } + readActionUndispatched { SymbolsProvider.extractSymbols(content) }.forEach { symbolsChannel.send(it) } } } } diff --git a/plugins/search-everywhere-ml/semantics/java/intellij.searchEverywhereMl.semantics.java.iml b/plugins/search-everywhere-ml/semantics/java/intellij.searchEverywhereMl.semantics.java.iml index 999f1a78543b..8f9dd3411ae0 100644 --- a/plugins/search-everywhere-ml/semantics/java/intellij.searchEverywhereMl.semantics.java.iml +++ b/plugins/search-everywhere-ml/semantics/java/intellij.searchEverywhereMl.semantics.java.iml @@ -12,6 +12,6 @@ - + \ No newline at end of file diff --git a/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaClassesProvider.kt b/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaClassesProvider.kt index 3db43b151da4..acf141979779 100644 --- a/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaClassesProvider.kt +++ b/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaClassesProvider.kt @@ -3,14 +3,25 @@ package com.intellij.searchEverywhereMl.semantics.java import com.intellij.platform.ml.embeddings.indexer.ClassesProvider import com.intellij.platform.ml.embeddings.indexer.entities.IndexableClass import com.intellij.platform.ml.embeddings.jvm.indices.EntityId -import com.intellij.psi.PsiFile -import com.intellij.psi.PsiJavaFile +import com.intellij.psi.impl.source.JavaLightTreeUtil +import com.intellij.psi.impl.source.tree.JavaElementType +import com.intellij.util.indexing.FileContent +import com.intellij.util.indexing.PsiDependentFileContent internal class JavaClassesProvider : ClassesProvider { - override fun extract(file: PsiFile): List { - return (file as PsiJavaFile).classes.asSequence() - .mapNotNull { c -> c.name } + override fun extract(fileContent: FileContent): List { + val ast = (fileContent as PsiDependentFileContent).lighterAST + val topLevelNodes = ast.getChildren(ast.root).asSequence() + val classes = topLevelNodes.filter { node -> node.tokenType == JavaElementType.CLASS } + + val classNames = classes + .mapNotNull { clazz -> JavaLightTreeUtil.getNameIdentifierText(ast, clazz) } + .filter { name -> name != ANONYMOUS_ID } + + return classNames .map { name -> IndexableClass(EntityId(name)) } .toList() } -} \ No newline at end of file +} + +private const val ANONYMOUS_ID = "" diff --git a/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaSymbolsProvider.kt b/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaSymbolsProvider.kt index ca71d15da3b0..2bfe3af1f82c 100644 --- a/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaSymbolsProvider.kt +++ b/plugins/search-everywhere-ml/semantics/java/src/com/intellij/searchEverywhereMl/semantics/java/JavaSymbolsProvider.kt @@ -3,15 +3,27 @@ package com.intellij.searchEverywhereMl.semantics.java import com.intellij.platform.ml.embeddings.indexer.SymbolsProvider import com.intellij.platform.ml.embeddings.indexer.entities.IndexableSymbol import com.intellij.platform.ml.embeddings.jvm.indices.EntityId -import com.intellij.psi.PsiFile -import com.intellij.psi.PsiJavaFile +import com.intellij.psi.impl.source.JavaLightTreeUtil +import com.intellij.psi.impl.source.tree.JavaElementType +import com.intellij.util.indexing.FileContent +import com.intellij.util.indexing.PsiDependentFileContent internal class JavaSymbolsProvider : SymbolsProvider { - override fun extract(file: PsiFile): List { - return (file as PsiJavaFile).classes.asSequence() - .flatMap { c -> c.methods.asSequence() } - .filter { m -> m.name != ANONYMOUS_ID } - .map { m -> IndexableSymbol(EntityId(m.name)) } + override fun extract(fileContent: FileContent): List { + val ast = (fileContent as PsiDependentFileContent).lighterAST + val topLevelNodes = ast.getChildren(ast.root).asSequence() + val classes = topLevelNodes.filter { node -> node.tokenType == JavaElementType.CLASS } + + val methods = classes + .flatMap { classNode -> ast.getChildren(classNode) } + .filter { node -> node.tokenType == JavaElementType.METHOD } + + val methodNames = methods + .mapNotNull { method -> JavaLightTreeUtil.getNameIdentifierText(ast, method) } + .filter { name -> name != ANONYMOUS_ID } + + return methodNames + .map { name -> IndexableSymbol(EntityId(name)) } .toList() } } diff --git a/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinClassesProvider.kt b/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinClassesProvider.kt index 4e0826c11d31..7b732a65bae9 100644 --- a/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinClassesProvider.kt +++ b/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinClassesProvider.kt @@ -3,13 +3,13 @@ package com.intellij.searchEverywhereMl.semantics.kotlin import com.intellij.platform.ml.embeddings.indexer.ClassesProvider import com.intellij.platform.ml.embeddings.indexer.entities.IndexableClass import com.intellij.platform.ml.embeddings.jvm.indices.EntityId -import com.intellij.psi.PsiFile import com.intellij.psi.util.childrenOfType +import com.intellij.util.indexing.FileContent import org.jetbrains.kotlin.psi.KtClass internal class KotlinClassesProvider : ClassesProvider { - override fun extract(file: PsiFile): List { - return file.childrenOfType().asSequence() + override fun extract(fileContent: FileContent): List { + return fileContent.psiFile.childrenOfType().asSequence() .mapNotNull { c -> c.name } .filter { name -> name != ANONYMOUS_ID } .map { name -> IndexableClass(EntityId(name)) } diff --git a/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinSymbolsProvider.kt b/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinSymbolsProvider.kt index 07f26800e185..4bde7bf0874e 100644 --- a/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinSymbolsProvider.kt +++ b/plugins/search-everywhere-ml/semantics/kotlin/src/com/intellij/searchEverywhereMl/semantics/kotlin/KotlinSymbolsProvider.kt @@ -3,13 +3,13 @@ package com.intellij.searchEverywhereMl.semantics.kotlin import com.intellij.platform.ml.embeddings.indexer.SymbolsProvider import com.intellij.platform.ml.embeddings.indexer.entities.IndexableSymbol import com.intellij.platform.ml.embeddings.jvm.indices.EntityId -import com.intellij.psi.PsiFile import com.intellij.psi.util.PsiTreeUtil +import com.intellij.util.indexing.FileContent import org.jetbrains.kotlin.psi.KtFunction internal class KotlinSymbolsProvider : SymbolsProvider { - override fun extract(file: PsiFile): List { - return PsiTreeUtil.findChildrenOfAnyType(file, false, KtFunction::class.java).asSequence() + override fun extract(fileContent: FileContent): List { + return PsiTreeUtil.findChildrenOfAnyType(fileContent.psiFile, false, KtFunction::class.java).asSequence() .mapNotNull { f -> f.name } .filter { name -> name != ANONYMOUS_ID } .map { name -> IndexableSymbol(EntityId(name)) } diff --git a/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonClassesProvider.kt b/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonClassesProvider.kt index 5f97f8b16902..f8fb9e6f13f5 100644 --- a/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonClassesProvider.kt +++ b/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonClassesProvider.kt @@ -3,13 +3,13 @@ package com.intellij.searchEverywhereMl.semantics.python import com.intellij.platform.ml.embeddings.indexer.ClassesProvider import com.intellij.platform.ml.embeddings.indexer.entities.IndexableClass import com.intellij.platform.ml.embeddings.jvm.indices.EntityId -import com.intellij.psi.PsiFile import com.intellij.psi.util.childrenOfType +import com.intellij.util.indexing.FileContent import com.jetbrains.python.psi.PyClass internal class PythonClassesProvider : ClassesProvider { - override fun extract(file: PsiFile): List { - return file.childrenOfType().asSequence() + override fun extract(fileContent: FileContent): List { + return fileContent.psiFile.childrenOfType() .mapNotNull { c -> c.name } .filter { name -> name != ANONYMOUS_ID } .map { name -> IndexableClass(EntityId(name)) } diff --git a/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonSymbolsProvider.kt b/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonSymbolsProvider.kt index 825ca8f83ffb..130c751b285a 100644 --- a/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonSymbolsProvider.kt +++ b/plugins/search-everywhere-ml/semantics/python/src/com.intellij.searchEverywhereMl.semantics.python/PythonSymbolsProvider.kt @@ -3,13 +3,13 @@ package com.intellij.searchEverywhereMl.semantics.python import com.intellij.platform.ml.embeddings.indexer.SymbolsProvider import com.intellij.platform.ml.embeddings.indexer.entities.IndexableSymbol import com.intellij.platform.ml.embeddings.jvm.indices.EntityId -import com.intellij.psi.PsiFile import com.intellij.psi.util.PsiTreeUtil +import com.intellij.util.indexing.FileContent import com.jetbrains.python.psi.PyFunction internal class PythonSymbolsProvider : SymbolsProvider { - override fun extract(file: PsiFile): List { - return PsiTreeUtil.findChildrenOfAnyType(file, false, PyFunction::class.java).asSequence() + override fun extract(fileContent: FileContent): List { + return PsiTreeUtil.findChildrenOfAnyType(fileContent.psiFile, false, PyFunction::class.java).asSequence() .mapNotNull { f -> f.name } .filter { name -> name != ANONYMOUS_ID } .map { name -> IndexableSymbol(EntityId(name)) }