From 5887fe82210cd69653805237c8a0cb1c1f591b56 Mon Sep 17 00:00:00 2001 From: Roman Ivanov Date: Thu, 28 Mar 2024 09:22:28 +0100 Subject: [PATCH] [java] implement constant pool parser and build id index using it for class files in jars IDEA-327594 GitOrigin-RevId: fff7c155b86e950e547fd8d01f142867c03c7c5c --- java/java-impl/src/META-INF/JavaPlugin.xml | 10 +- .../JavaBinaryFileSourceProvider.java | 19 + .../impl/idCache/ConstantPoolParser.java | 110 ++++ .../JavaIdIndexRegistryValueListener.kt | 16 + .../cache/impl/idCache/JavaIdIndexer.java | 82 ++- .../cache/impl/idCache/JvmIdentifierUtil.java | 113 ++++ .../constantPool/AnnotationDeclaration.class | Bin 0 -> 399 bytes .../psi/constantPool/Dollar$InClass.class | Bin 0 -> 365 bytes .../constantPool/DoubleAndLongFields.class | Bin 0 -> 374 bytes .../MethodWithExtendsWildcard.class | Bin 0 -> 579 bytes .../MethodWithSuperWildcard.class | Bin 0 -> 573 bytes .../constantPool/MethodWithTypeParam.class | Bin 0 -> 534 bytes .../MethodWithTypeParamExtends.class | Bin 0 -> 574 bytes .../psi/constantPool/MethodWithWildcard.class | Bin 0 -> 510 bytes .../testData/psi/constantPool/MyEnum.class | Bin 0 -> 856 bytes .../psi/constantPool/RuntimeAnnotation.class | Bin 0 -> 427 bytes .../RuntimeAnnotationWithArgs.class | Bin 0 -> 588 bytes .../psi/constantPool/SeveralBounds.class | Bin 0 -> 456 bytes .../constantPool/SeveralBoundsInMethod.class | Bin 0 -> 450 bytes .../testData/psi/constantPool/Simple.class | Bin 0 -> 380 bytes .../testData/psi/constantPool/TryCatch.class | Bin 0 -> 428 bytes .../constantPool/TypeParameterInClass.class | Bin 0 -> 446 bytes .../psi/constantPool/WithParameter.class | Bin 0 -> 806 bytes .../psi/impl/ConstantPoolParserTest.kt | 545 ++++++++++++++++++ .../psi/impl/JvmIdentifierUtilTest.java | 45 ++ platform/indexing-api/api-dump-unreviewed.txt | 3 + .../resources/META-INF/Indexing.xml | 1 + .../indexing/BinaryFileSourceProvider.java | 27 + .../impl/cache/impl/BaseFilterLexerUtil.java | 4 +- .../impl/cache/impl/id/IdDataConsumer.java | 1 - .../psi/impl/search/PsiSearchHelperImpl.java | 65 ++- .../com/intellij/psi/impl/search/helper.kt | 8 +- platform/util/api-dump-unreviewed.txt | 1 + .../openapi/util/text/StringUtil.java | 56 ++ .../intellij/util/text/StringUtilTest.java | 17 + 35 files changed, 1093 insertions(+), 30 deletions(-) create mode 100644 java/java-impl/src/com/intellij/codeEditor/JavaBinaryFileSourceProvider.java create mode 100644 java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/ConstantPoolParser.java create mode 100644 java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexRegistryValueListener.kt create mode 100644 java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JvmIdentifierUtil.java create mode 100644 java/java-tests/testData/psi/constantPool/AnnotationDeclaration.class create mode 100644 java/java-tests/testData/psi/constantPool/Dollar$InClass.class create mode 100644 java/java-tests/testData/psi/constantPool/DoubleAndLongFields.class create mode 100644 java/java-tests/testData/psi/constantPool/MethodWithExtendsWildcard.class create mode 100644 java/java-tests/testData/psi/constantPool/MethodWithSuperWildcard.class create mode 100644 java/java-tests/testData/psi/constantPool/MethodWithTypeParam.class create mode 100644 java/java-tests/testData/psi/constantPool/MethodWithTypeParamExtends.class create mode 100644 java/java-tests/testData/psi/constantPool/MethodWithWildcard.class create mode 100644 java/java-tests/testData/psi/constantPool/MyEnum.class create mode 100644 java/java-tests/testData/psi/constantPool/RuntimeAnnotation.class create mode 100644 java/java-tests/testData/psi/constantPool/RuntimeAnnotationWithArgs.class create mode 100644 java/java-tests/testData/psi/constantPool/SeveralBounds.class create mode 100644 java/java-tests/testData/psi/constantPool/SeveralBoundsInMethod.class create mode 100644 java/java-tests/testData/psi/constantPool/Simple.class create mode 100644 java/java-tests/testData/psi/constantPool/TryCatch.class create mode 100644 java/java-tests/testData/psi/constantPool/TypeParameterInClass.class create mode 100644 java/java-tests/testData/psi/constantPool/WithParameter.class create mode 100644 java/java-tests/testSrc/com/intellij/psi/impl/ConstantPoolParserTest.kt create mode 100644 java/java-tests/testSrc/com/intellij/psi/impl/JvmIdentifierUtilTest.java create mode 100644 platform/indexing-api/src/com/intellij/platform/indexing/BinaryFileSourceProvider.java diff --git a/java/java-impl/src/META-INF/JavaPlugin.xml b/java/java-impl/src/META-INF/JavaPlugin.xml index 749638d127d8..af4d1aa3e758 100644 --- a/java/java-impl/src/META-INF/JavaPlugin.xml +++ b/java/java-impl/src/META-INF/JavaPlugin.xml @@ -306,6 +306,8 @@ + + @@ -1007,6 +1009,7 @@ + @@ -1267,6 +1270,7 @@ + @@ -2515,8 +2519,8 @@ - + @@ -2550,6 +2554,8 @@ + diff --git a/java/java-impl/src/com/intellij/codeEditor/JavaBinaryFileSourceProvider.java b/java/java-impl/src/com/intellij/codeEditor/JavaBinaryFileSourceProvider.java new file mode 100644 index 000000000000..40e4b882c2ae --- /dev/null +++ b/java/java-impl/src/com/intellij/codeEditor/JavaBinaryFileSourceProvider.java @@ -0,0 +1,19 @@ +// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.codeEditor; + +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.platform.indexing.BinaryFileSourceProvider; +import com.intellij.psi.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public final class JavaBinaryFileSourceProvider implements BinaryFileSourceProvider { + + @Override + public @Nullable PsiFile findSourceFile(@NotNull PsiBinaryFile file) { + VirtualFile virtualFile = JavaEditorFileSwapper.findSourceFile(file.getProject(), file.getVirtualFile()); + PsiManager manager = file.getManager(); + if (virtualFile == null) return null; + return manager.findFile(virtualFile); + } +} \ No newline at end of file diff --git a/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/ConstantPoolParser.java b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/ConstantPoolParser.java new file mode 100644 index 000000000000..a7745fa4e9fc --- /dev/null +++ b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/ConstantPoolParser.java @@ -0,0 +1,110 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.impl.cache.impl.idCache; + +import com.intellij.openapi.util.text.StringUtil; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +import java.io.DataInput; +import java.io.IOException; + +@ApiStatus.Internal +public final class ConstantPoolParser { + private final @NotNull DataInput myInput; + private final @NotNull Callback myCallback; + + public ConstantPoolParser(@NotNull DataInput input, @NotNull Callback callback) { + myInput = input; + myCallback = callback; + } + + public void parse() throws IOException, ClassFormatException { + int magic = myInput.readInt(); + if(magic != 0xCAFEBABE) { + throw new ClassFormatException("Invalid class file: magic incorrect"); + } + + // version + myInput.readShort(); + myInput.readShort(); + + int constantPoolCount = myInput.readShort(); + + for(int i = 0; i < constantPoolCount - 1;) { + i += readConstant(); + } + } + + /** + * @return number of slots occupied in constant pool + */ + @SuppressWarnings("DuplicateBranchesInSwitch") + private int readConstant() throws IOException, ClassFormatException { + int tag = myInput.readByte(); + switch (tag) { + case 1 -> { + String utf8String = myInput.readUTF(); + myCallback.onUtf8(utf8String); // Utf8 + } + case 3 -> // Integer + myInput.skipBytes(4); // Integer value + case 4 -> // Float + myInput.skipBytes(4); // Float value + case 5 -> {// Long + myInput.skipBytes(8); // Long value + return 2; + } + case 6 -> { // Double + myInput.skipBytes(8); // Double value + return 2; + } + case 7 -> // Class + myInput.skipBytes(2); // Class index + case 8 -> // String + myInput.skipBytes(2); // String index + case 9 -> { // Fieldref + myInput.skipBytes(4); // Class index + NameAndType index + } + case 10 -> { // Methodref + myInput.skipBytes(4); // Class index + NameAndType index + } + case 11 -> { // InterfaceMethodref + myInput.skipBytes(4); // Class index + NameAndType index + } + case 12 -> { // NameAndType + myInput.skipBytes(4); // Name index + Type index + } + case 15 -> { // MethodHandle + myInput.skipBytes(3); // Reference kind + Reference index + } + case 16 -> // MethodType + myInput.skipBytes(2); // Descriptor index + case 17 -> { // Dynamic + myInput.skipBytes(4); // Bootstrap method index + NameAndType index + } + case 18 -> { // InvokeDynamic + myInput.skipBytes(4); // Bootstrap method index + NameAndType index + } + case 19 -> // Module + myInput.skipBytes(2); // Name index + case 20 -> // Package + myInput.skipBytes(2); // Name index + default -> throw new ClassFormatException("Invalid constant pool entry tag: " + tag); + } + return 1; + } + + public interface Callback { + void onUtf8(String str); + } + + public static class ClassFormatException extends Exception { + ClassFormatException(String message) { + super(message); + } + } + + public static void main(String[] args) { + System.out.println("Is java id \uD83D\uDC69\u200D: " + StringUtil.isJavaIdentifier("\uD83D\uDC69\u200D")); + } +} diff --git a/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexRegistryValueListener.kt b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexRegistryValueListener.kt new file mode 100644 index 000000000000..6069c4861efb --- /dev/null +++ b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexRegistryValueListener.kt @@ -0,0 +1,16 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.impl.cache.impl.idCache + +import com.intellij.openapi.util.registry.RegistryValue +import com.intellij.openapi.util.registry.RegistryValueListener +import com.intellij.psi.impl.cache.impl.id.IdIndex +import com.intellij.util.indexing.FileBasedIndex + +class JavaIdIndexRegistryValueListener : RegistryValueListener { + override fun afterValueChanged(value: RegistryValue) { + if (JavaIdIndexer.ENABLED_REGISTRY_KEY == value.key) { + JavaIdIndexer.isEnabled = value.asBoolean() + FileBasedIndex.getInstance().requestRebuild(IdIndex.NAME) + } + } +} \ No newline at end of file diff --git a/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexer.java b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexer.java index 31b2495acbba..ac932f395755 100644 --- a/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexer.java +++ b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexer.java @@ -1,17 +1,89 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.psi.impl.cache.impl.idCache; +import com.intellij.ide.highlighter.JavaClassFileType; import com.intellij.lang.java.JavaParserDefinition; import com.intellij.lexer.Lexer; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.util.registry.Registry; +import com.intellij.openapi.vfs.VirtualFile; import com.intellij.pom.java.LanguageLevel; import com.intellij.psi.impl.cache.impl.OccurrenceConsumer; -import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer; +import com.intellij.psi.impl.cache.impl.id.IdDataConsumer; +import com.intellij.psi.impl.cache.impl.id.IdIndexEntry; +import com.intellij.psi.impl.cache.impl.id.IdIndexer; +import com.intellij.psi.impl.source.JavaFileElementType; +import com.intellij.psi.search.UsageSearchContext; +import com.intellij.util.indexing.FileContent; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.Map; + +import static com.intellij.psi.impl.cache.impl.BaseFilterLexerUtil.scanContentWithCheckCanceled; + +public final class JavaIdIndexer implements IdIndexer { + private final static Logger LOG = Logger.getInstance(JavaIdIndexer.class); + static final String ENABLED_REGISTRY_KEY = "index.ids.from.java.sources.in.jar"; + static volatile boolean isEnabled = Registry.is("index.ids.from.java.sources.in.jar", true); -public final class JavaIdIndexer extends LexerBasedIdIndexer { @Override - public @NotNull Lexer createLexer(final @NotNull OccurrenceConsumer consumer) { - return createIndexingLexer(consumer); + public @NotNull Map map(@NotNull FileContent inputData) { + + VirtualFile file = inputData.getFile(); + String extension = file.getExtension(); + if (extension != null && extension.equals(JavaClassFileType.DEFAULT_EXTENSION) && isEnabled) { + Map idEntries = calculateIdEntriesParsingConstantPool(inputData); + if (idEntries != null) return idEntries; + return Map.of(); + } + // we are skipping indexing of sources in libraries (we are going to index only the compiled library classes) + if (!isEnabled && !JavaFileElementType.isInSourceContent(file)) { + return Map.of(); + } + + IdDataConsumer consumer = new IdDataConsumer(); + scanContentWithCheckCanceled(inputData, createIndexingLexer(new OccurrenceConsumer(consumer, false))); + return consumer.getResult(); + } + + @Nullable + private static Map calculateIdEntriesParsingConstantPool(@NotNull FileContent inputData) { + IdDataConsumer consumer = new IdDataConsumer(); + // optimised to avoid creation of list for every UTF8 entry + IntList startInclList = new IntArrayList(); + IntList endExclList = new IntArrayList(); + ConstantPoolParser parser = new ConstantPoolParser(new DataInputStream(new ByteArrayInputStream(inputData.getContent())), str -> { + ProgressManager.checkCanceled(); + if (str.isEmpty()) return; + // it is likely a method signature, we don't need to handle it because all its parts will it is possible to find in the other entries. + if (str.charAt(0) == '(') return; + + if (JvmIdentifierUtil.collectJvmIdentifiers(str, (sequence, startIncl, endExcl) -> { + startInclList.add(startIncl); + endExclList.add(startIncl); + })) { + for (int i = 0; i < startInclList.size(); i++) { + consumer.addOccurrence(str, startInclList.getInt(i), endExclList.getInt(i), UsageSearchContext.IN_CODE); + } + } + startInclList.clear(); + endExclList.clear(); + }); + try { + parser.parse(); + } + catch (IOException | ConstantPoolParser.ClassFormatException e) { + LOG.warn("Exception while handling file " + inputData.getFileName(), e); + return null; + } + return consumer.getResult(); } public static Lexer createIndexingLexer(OccurrenceConsumer consumer) { diff --git a/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JvmIdentifierUtil.java b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JvmIdentifierUtil.java new file mode 100644 index 000000000000..3f59b902683a --- /dev/null +++ b/java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JvmIdentifierUtil.java @@ -0,0 +1,113 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.impl.cache.impl.idCache; + +import com.intellij.openapi.util.text.StringUtil; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; + +public final class JvmIdentifierUtil { + private JvmIdentifierUtil() { } + + public interface IdConsumer { + /** + * Handles valid JVM identifier extracted from the string + * Warning: Implementation must be ready for rollback - several valid identifiers may be fed to {@code accept} before checking that there + * is an invalid one present in this string and all previous ones should be rolled back. + * @param sequence original sequence, from which the identifiers are extracted. Args correspond to offsets in the sequence. + */ + void accept(@NotNull CharSequence sequence, int startIncl, int endExcl); + } + + /** + * Collects JVM identifiers from a given string and passes them to the provided IdConsumer. + * + * @param str The string to collect JVM identifiers from. + * @param partsConsumer The IdConsumer that will receive the collected JVM identifiers. + * @return true if JVM identifiers are collected successfully, false otherwise. + */ + @Contract(mutates = "param2") + public static boolean collectJvmIdentifiers(@NotNull String str, @NotNull IdConsumer partsConsumer) { + if (str.isEmpty()) return false; + + if (str.charAt(str.length() - 1) == ';' || str.charAt(0) == '(') { + // probably field descriptor, e.g. [[Ljava/lang/String; or [B or S + // also it may be Signature, e.g. + // Ljava/util/Map<[Ljava/lang/String;Ljava/util/List;>;>;>; + // It doesn't make sense to take the strings out of it recursively, so instead parts between 'L' and (';' or '<') are searched + int current = 0; + int start; + outer: + while (current < str.length()) { + char firstChar = str.charAt(current); + if (firstChar == 'L') { + current++; + start = current; + while (current < str.length()) { + char ch = str.charAt(current); + current++; + if (ch == ';' || ch == '<') { + if (!collectSlashSeparated(str, start, current - 1, partsConsumer)) { + return false; + } + continue outer; + } + } + } + current++; + } + if (current == str.length()) { + return true; + } + } + + return collectSlashSeparated(str, 0, str.length(), partsConsumer); + } + + private static int indexOfSlash(String str, int startIndexIncl, int endIndexExcl) { + int current = startIndexIncl; + while (current < endIndexExcl) { + char ch = str.charAt(current); + if (ch == '/') { + return current; + } + + current++; + } + return -1; + } + + private static boolean collectSlashSeparated(@NotNull String str, int startIncl, int endExcl, @NotNull IdConsumer partsConsumer) { + int partStartIncl = startIncl; + while (partStartIncl < endExcl) { + int indexOfSlash = indexOfSlash(str, partStartIncl, endExcl); + if (indexOfSlash == -1) { + break; + } + if (partStartIncl == indexOfSlash) { + return false; + } + if (!StringUtil.isJavaIdentifier(str, partStartIncl, indexOfSlash)) { + return false; + } + partsConsumer.accept(str, partStartIncl, indexOfSlash); + partStartIncl = indexOfSlash + 1; + if (partStartIncl == endExcl) { // '/' is at the end + return false; + } + } + // handling rest + if (partStartIncl == 0) { + if (!StringUtil.isJavaIdentifier(str, 0, str.length())) { + return false; + } + partsConsumer.accept(str, partStartIncl, endExcl); + } else { + if (!StringUtil.isJavaIdentifier(str, partStartIncl, endExcl)) { + return false; + } + partsConsumer.accept(str, partStartIncl, endExcl); + } + return true; + } + +} diff --git a/java/java-tests/testData/psi/constantPool/AnnotationDeclaration.class b/java/java-tests/testData/psi/constantPool/AnnotationDeclaration.class new file mode 100644 index 0000000000000000000000000000000000000000..88a78a28a0d180850aae92c681aaf19b5e80ed2f GIT binary patch literal 399 zcmZ{gzfJ-{5XQd&WaU5wwZTSA)M&#sb~F&wgakuDVn3cMIb8P4?HviPX2ApSP{!eo z2!#|gn{WPn^UJP&zLx+VaHepgP*-RuoD#|}izl1T2(^K=I_?{GPCG)?C2?pcuWV9H zIKR6ePI=D0VLS21S*t^6^sJ2%PQu7O>JLs$La5Ict@m8C=J|F-%Y}{8jW~UA@agZa z28QFz?9fpB734g+oylQAGz&3RTqBEgWU}7-e}q87WB|beq3M!fyxw literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/Dollar$InClass.class b/java/java-tests/testData/psi/constantPool/Dollar$InClass.class new file mode 100644 index 0000000000000000000000000000000000000000..609d6b28a07a1e67cd6c8ff2d231f733ad542091 GIT binary patch literal 365 zcmZ8d%TB{E5S&fZri9Rj_6v}5;31IthGt2ICMOIsyC$UGxa?TD*!>3-g@Li?u8pLhnu))l3MT^NVGG zK4EZgwH9@fnW+|=4WXYXGgDGm1j-2h)K-!(%$1Rk+m}MtPomIV8RfPVdMRq<_;1l~ zR*Ey?d_VP;1FH6(%p?!8Sy@gG7j|2ha;99`c;9gCo{lpFJP|;=ju0|>eCF90kC~;s uyI%M59rV_=BSzPDrW3~KM}Pr_%z*gMe8sHQ9UA?AM*g4HNH}G=htW4HjX&Q2 literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/DoubleAndLongFields.class b/java/java-tests/testData/psi/constantPool/DoubleAndLongFields.class new file mode 100644 index 0000000000000000000000000000000000000000..7207b3b1ef77fbd5233f5543e9756d56af3decd9 GIT binary patch literal 374 zcmZusO;5r=6r3%E+EM{2D1KZ_JSYdbc+(h-$iXyv&~V?j8(dP`q_qAo55&ZSKfoVl zoCS|;GVi^ao%fRY`ThC^aDgKaF7_HXCuDpswlJu#XV`}AJ2-G~C?FDIOr*qvK;ueh zy1W*sot{kv?7KWs0_{*|>alu@RPiFClu9?uW0_86p$-2R?ebMG1p47Puc+Q;NtkD| ziB@T{xD@c7^Qwr|M4JPH4W9G4e3w32X#4Qc^xwMCOzw8@*CyJY@)4FP(q Jzdoyl-9O{iL4yDQ literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/MethodWithExtendsWildcard.class b/java/java-tests/testData/psi/constantPool/MethodWithExtendsWildcard.class new file mode 100644 index 0000000000000000000000000000000000000000..367de0de81b63c82d551e3b56947ef262e9a659b GIT binary patch literal 579 zcmaJ;%T59@6g`~>2%;b;YD`Srp(f169fS>0H=`~vur4zVQ!>op&@uX1u1s9`0e+P6 zwqT4T+Qsd?r{~;PKR(~y0bJmq3>$kzG>h10D0F2aRgb}LoemkCei{mfsxK39o6p7~ zyXWJWkh-4+JRb5)>Uae@Dv}F^BmYLIC=EwaMVC)XB;jHt<1pY^*kQ0_$k1r{Q~t=k zT*=t;l~5CzL?KsEpTUw5L#yl0`6Ku8 zOeUT$*I8fpa#vjO1w&)=kfHs%+tcd1NNa}{&`=2>Ax3Quy~qV)WtL+DOW=;2hl!tMWn$q6_)*5$ zg#;D0n4NiV-kaI|`u_L?&_p$h1j=bt(l{Zc+A2_Shmfe94G780a3l$NM+NeBIUmZ% z6T=xJMJM#dY#<_~>ou5+C+e9{c6u_Ngrf%)PrTQKbVVfQErO*+OxB#KcoB`62*!=O z;Z*u@OHZ;Fx=R~x(S*w1Y3#Y4nDe|x_v*>*-gkREA=6jmK*Y<4H>|fe)Wx=Gr}kT_ zH^}zGW#r3irQcWG@r1hGEQcg)LTLw2DE!|Yr?B{Y^|8R;37cbzs|Mf1w|)K&dNXvI zW6FSpJ1 cSPE6Ng>AA0LYGAqQY2(2E>xmuH;(px0ZcD`O#lD@ literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/MethodWithWildcard.class b/java/java-tests/testData/psi/constantPool/MethodWithWildcard.class new file mode 100644 index 0000000000000000000000000000000000000000..fc08483b8bdc8e0d0c44f2e66f16344f8e6393cc GIT binary patch literal 510 zcmZWlO;5r=5PjPZDu{xB#j7{uU@zVvTuk&*^?-r%(l)T9v}D-_`34K5n6aqhwQ&}XQPMJCjQ!5&=A7@XTYP8eFD$dbomvqGAJ9=Nv190k4~bgk@m#3YT%>WtWjtcAM9k0~gloR%L7_w%gkq;g z+Q!{-$u|t`YLmhLZ}hVLHS1QmQk}IxGE&g=vTcoMcXx|c_E|ZPUubghr?d$ z5u0$}(xQiLn$zr%@@yGAE!Ih-a0!dPQ6CHRZEz{ps0!$2bi2bhuvgReDAo*UE+QU49eE<+2Hko!=Sy%0hLo8B;JgnYT1Rb zygz;GG1Lcs=p81Lkr%z^lBEo+qGRADng*6oGf+p3p_WfA2pGz4 zFb%2URm(1rAB|2tHzv}j#c}onKAZIlu)oG#?%Lby(#;z5J4kzLf*hco_<`ky5K9>u z8cbXPG1F<2cCqw1*k_b3Abq40!!>$W(i}NQ16^O>cTh^Bx+%B5pnL&Y;7gbAHz;Cc zD6FB8=E)603C1oYdH|)h-8@I-Q<^T)<>d-)3F)|f*q~A;EhpwNV>+t2M#D*IMSQR1}3Q2>olN zmAN6<7nc#Cx5{26VHhecAIeQE^A{1PtW3fz5osiHRmF|iGw+HS)36>mrKuy|?JA->FIyTegX$AB~c0)4Vz!DHKJ8CM7`m@)&Y z8LF*BF)-Z7K)pyaXoP)zPIga06n4dpjAeEgGEav5l`~YPgtpX@jHt({BKg!8`_vL$ zvdj)6Z6@O%{8KaeMKOOomold=mXJ%DNZe=hFcjlHJ)f(-_s|)Nu3^|q%;(0h&2HgO zfAf^282hm38smr2qE;v_Q|66xAbtD+_D+BWmtuxsz(Jm(JAO}N1qI4%6e(I*MTyQl beOsfwF0g@3Dg(h5Qnb@oWfHKEt!IA${^6QU literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/SeveralBounds.class b/java/java-tests/testData/psi/constantPool/SeveralBounds.class new file mode 100644 index 0000000000000000000000000000000000000000..6d238e67b1879ad4d074b755397a2e09d91d4510 GIT binary patch literal 456 zcmY*Vu};G<5Pfc%hL)C;Qn9lE16i2bC=ygcLW*EW)ZJ~Z;3{=0*Qvy3F(EPV0elqV z9EwP6>F(b1J3srrzCS(z++y5A3yz0&A1*o!!!_UYAm=Izo)>GeGz^_kDrq7H=kjXq z;fP_7h`rD}zb{I)+A*|e#Y!;vu~g!z+$@An`64G~6c<4~SZleGI literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/SeveralBoundsInMethod.class b/java/java-tests/testData/psi/constantPool/SeveralBoundsInMethod.class new file mode 100644 index 0000000000000000000000000000000000000000..76565a0ed199217d303f1973df8bb511e9e113d4 GIT binary patch literal 450 zcmZut%T5A85UgHaf+&bFCf>aHNaW&)B^)F%AzAgXg!{0xFl3q0VVD2%Wa7aO@S}`9 zNTNpO&^^^v)!j26pKtE~u5nmHfIM?Npg*`(ZriBqNl&+UTrDD4d;- zD%c?$#k^vt(i@wb$#P)sIGfoCq1d+*CNv^#_Z*$ve=l}anJiAqPxX^%I>Vm8xD z!f~{mx8?RpRxSQNE>5AgaMjjI9T6WUrktGNdN!< literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/Simple.class b/java/java-tests/testData/psi/constantPool/Simple.class new file mode 100644 index 0000000000000000000000000000000000000000..29ab0a22c304be715604e8ac58974b88ec379517 GIT binary patch literal 380 zcmYjMyH3ME5S(@Vifss(R|_2ki8M3_M1w?SL1C2s>>SBOXB!y@iO-@yNECbkABEVv zM1(C(GMN5H}*g|M|=^1hhMvcnM{KsY+#cSF!-}MWn@OsTI}0`|4&FO KA@9jl2iw2zxju6M literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/TryCatch.class b/java/java-tests/testData/psi/constantPool/TryCatch.class new file mode 100644 index 0000000000000000000000000000000000000000..1ec3a1d7c39897cb4668323cc935adb1007314ad GIT binary patch literal 428 zcmYLEJx>Bb5Pf?b?13PNpRu!33Tf;#HbiYsO@z?zovSRu9XS@GzsQJi;>I?Xt^j-_pLzC#6 z**MZ!Pemgl%`lDCsIM~P-V4yT12Ywt*#ZT92k~2Avg-B)O zcGFoF>r3NAYs^Z{ROM!{HYeL&2acqr4Ca*+RGUi~9rjhQ>O|K=Vm9+7hE9o`%K raGq0r5(+dY)Z|?7I!y8h0c|B(UWl%fBjH_1)SUu%B_Oat@1gk%oL)uo literal 0 HcmV?d00001 diff --git a/java/java-tests/testData/psi/constantPool/TypeParameterInClass.class b/java/java-tests/testData/psi/constantPool/TypeParameterInClass.class new file mode 100644 index 0000000000000000000000000000000000000000..c819ad8fc034b5a581fbd60773acfd0a2095d216 GIT binary patch literal 446 zcmZut!AiqG5PjP=iLKGrS_F@R2lZet9!gURp(qGZiwy-&vWZK(CEe&I5&SGqf(Jjq zj}j-bPz%|GH~Zeqo1NY7pU*D<7w9xl#h!(>g?$EdAeBsq4AqmKOR` z!}#)X?B2UK&XD}|iJU2(W|{`*4c5ti-&^ok-i^4Lbw_iqC*ma&DiqG}U*axUh%lu# zjY*v8P+ZCUA%|tNp6Bgs)L<}l$_f}->l^pi}YFuVC8X>GL+}apyep zIk#QzkL;dS(jT=Qi8eKd@q=(p*WY}0GH-+ zVrEf53TcMgpVMO~{mkrY@F<2Fhe*)-F<%n&UXZ3WLzYcDqdk563id)u0$Ex!A(%yu z^g`qS3wg3EY=np*H%Scn7cmzjGHhZiBoF6CnjSVtrg-nIhIMH8ce7PSvw>N L>{9M(sO^0L!E(Rq literal 0 HcmV?d00001 diff --git a/java/java-tests/testSrc/com/intellij/psi/impl/ConstantPoolParserTest.kt b/java/java-tests/testSrc/com/intellij/psi/impl/ConstantPoolParserTest.kt new file mode 100644 index 000000000000..6ecbbfb1dff3 --- /dev/null +++ b/java/java-tests/testSrc/com/intellij/psi/impl/ConstantPoolParserTest.kt @@ -0,0 +1,545 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.impl + +import com.intellij.psi.impl.cache.impl.idCache.ConstantPoolParser +import com.intellij.psi.impl.cache.impl.idCache.JvmIdentifierUtil.collectJvmIdentifiers +import com.intellij.testFramework.fixtures.LightJavaCodeInsightFixtureTestCase5 +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream +import java.io.DataInputStream +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.io.path.readBytes + +class ConstantPoolParserTest : LightJavaCodeInsightFixtureTestCase5() { + private val constantPoolPath: Path by lazy { Paths.get(fixture.testDataPath).resolve("psi").resolve("constantPool") } + + /** + * Source: + * ``` + * public class Simple { + * String field = "my field"; + * + * void method() { + * + * } + * } + * + * ``` + */ + @Test fun testSimple() = doTest(""" + Code + LineNumberTable + LocalVariableTable + Object + Simple + SourceFile + String + field + java + lang + method + this + """.trimIndent()) + + /** + * Source: + * ``` + * public class WithParameter { + * public static void foo(String[] s) {} + * public static void bar(Map>>> s) {} + * } + * ``` + */ + @Test fun testWithParameter() = doTest(""" + Code + Comparator + LineNumberTable + List + LocalVariableTable + LocalVariableTypeTable + Map + Object + Signature + SourceFile + String + WithParameter + bar + example + foo + java + lang + org + s + this + util + """.trimIndent()) + + /** + * Source: + * ``` + * @RuntimeAnnotation.RuntimeAnn + * public class RuntimeAnnotation { + * @Retention(RetentionPolicy.RUNTIME) + * @Target(ElementType.TYPE) + * @interface RuntimeAnn { + * + * } + * } + * + * ``` + * + */ + @Test fun testRuntimeAnnotation() = doTest(""" + Code + InnerClasses + LineNumberTable + LocalVariableTable + Object + RuntimeAnn + RuntimeAnnotation + RuntimeAnnotation${'$'}RuntimeAnn + RuntimeVisibleAnnotations + SourceFile + java + lang + this + """.trimIndent()) + + /** + * Source: + * ``` + * @RuntimeAnnotationWithArgs.RuntimeAnn(justEnum = RuntimeAnnotationWithArgs.MyEnum.Variant2) + * public class RuntimeAnnotationWithArgs { + * @Retention(RetentionPolicy.RUNTIME) + * @Target(ElementType.TYPE) + * @interface RuntimeAnn { + * MyEnum enumWithDefault() default MyEnum.Variant1; + * MyEnum justEnum(); + * } + * + * enum MyEnum { + * Variant1, + * Variant2, + * } + * } + * ``` + */ + @Test fun testRuntimeAnnotationWithArgs() = doTest(""" + Code + InnerClasses + LineNumberTable + LocalVariableTable + MyEnum + Object + RuntimeAnn + RuntimeAnnotationWithArgs + RuntimeAnnotationWithArgs${'$'}MyEnum + RuntimeAnnotationWithArgs${'$'}RuntimeAnn + RuntimeVisibleAnnotations + SourceFile + Variant2 + java + justEnum + lang + this + """.trimIndent()) + + /** + * Source: + * ``` + * package pack; + * + * public @interface AnnotationDeclaration { + * String stringValue() default "my default value"; + * MyEnum enumValue(); + * + * enum MyEnum { + * Variant1, + * Variant2, + * } + * } + * ``` + */ + @Test fun testAnnotationDeclaration() = doTest(""" + Annotation + AnnotationDeclaration + AnnotationDeclaration${'$'}MyEnum + AnnotationDefault + InnerClasses + MyEnum + Object + SourceFile + String + annotation + enumValue + java + lang + pack + stringValue + """.trimIndent()) + + /** + * Source: + * ``` + *public enum MyEnum { + * Variant1, + * Variant2, + * } + * ``` + */ + @Test fun testMyEnum() = doTest(""" + ${'$'}VALUES + Class + Code + Enum + LineNumberTable + LocalVariableTable + MyEnum + Object + Signature + SourceFile + String + Variant1 + Variant2 + clone + java + lang + name + this + valueOf + values + """.trimIndent()) + + /** + * Source: + * ``` + * public class DoubleAndLongFields { + * double d = 10.0; + * long l = 4L; + * } + * ``` + */ + @Test fun testDoubleAndLongFields() = doTest(""" + Code + D + DoubleAndLongFields + J + LineNumberTable + LocalVariableTable + Object + SourceFile + d + java + l + lang + this + """.trimIndent()) + /** + * Source: + * ``` + * public class TryCatch { + * void foo() { + * try { + * bar(); + * } catch (RuntimeException e) { + * + * } + * } + * + * native void bar(); + * } + * + * ``` + */ + @Test fun testTryCatch() = doTest(""" + Code + LineNumberTable + LocalVariableTable + Object + RuntimeException + SourceFile + StackMapTable + TryCatch + bar + foo + java + lang + this + """.trimIndent()) + + /** + * Source: + * ``` + * import java.util.List; + * + * public class MethodWithExtendsWildcard { + * public void id(List param) {} + * } + * + * ``` + */ + @Test fun testMethodWithExtendsWildcard() { + doTest(""" + CharSequence + Code + LineNumberTable + List + LocalVariableTable + LocalVariableTypeTable + MethodWithExtendsWildcard + Object + Signature + SourceFile + id + java + lang + param + this + util + """.trimIndent()) + } + + /** + * Source: + * ``` + * import java.util.List; + * + * public class MethodWithSuperWildcard { + * public void id(List param) {} + * } + * + * ``` + */ + @Test fun testMethodWithSuperWildcard() { + doTest(""" + CharSequence + Code + LineNumberTable + List + LocalVariableTable + LocalVariableTypeTable + MethodWithSuperWildcard + Object + Signature + SourceFile + id + java + lang + param + this + util + """.trimIndent()) + } + + /** + * Source: + * ``` + * public class MethodWithTypeParam { + * public void id(TYPE_PARAM param) {} + * } + * ``` + */ + @Test fun testMethodWithTypeParam() { + doTest(""" + Code + LineNumberTable + LocalVariableTable + LocalVariableTypeTable + MethodWithTypeParam + Object + Signature + SourceFile + id + java + lang + param + this + """.trimIndent()) + } + + /** + * Source: + * ``` + * public class MethodWithTypeParamExtends { + * public void id(TYPE_PARAM param) {} + * } + * ``` + */ + @Test fun testMethodWithTypeParamExtends() { + doTest(""" + CharSequence + Code + LineNumberTable + LocalVariableTable + LocalVariableTypeTable + MethodWithTypeParamExtends + Object + Signature + SourceFile + id + java + lang + param + this + """.trimIndent()) + } + + /** + * Source: + * ``` + * import java.util.List; + * + * public class MethodWithWildcard { + * public void id(List param) {} + * } + * ``` + */ + @Test fun testMethodWithWildcard() { + doTest(""" + Code + LineNumberTable + List + LocalVariableTable + LocalVariableTypeTable + MethodWithWildcard + Object + Signature + SourceFile + id + java + lang + param + this + util + """.trimIndent()) + } + + /** + * Source: + * ``` + * public class TypeParameterInClass {} + * ``` + */ + @Test fun testTypeParameterInClass() { + doTest(""" + CharSequence + Code + LineNumberTable + LocalVariableTable + LocalVariableTypeTable + Object + Signature + SourceFile + TypeParameterInClass + java + lang + this + """.trimIndent()) + } + + /** + * Source: + * ``` + * import java.util.List; + * + * public class SeveralBounds & List> { + * } + * ``` + */ + @Test fun testSeveralBounds() { + doTest(""" + Code + Comparable + Integer + LineNumberTable + List + LocalVariableTable + LocalVariableTypeTable + Object + SeveralBounds + Signature + SourceFile + String + java + lang + this + util + """.trimIndent()) + } + + /** + * Source: + * ``` + * import java.util.List; + * + * public class SeveralBoundsInMethod { + * >void foo() {} + * } + * ``` + */ + @Test fun testSeveralBoundsInMethod() { + doTest(""" + Code + LineNumberTable + LocalVariableTable + Object + SeveralBoundsInMethod + Signature + SourceFile + foo + java + lang + this + """.trimIndent()) + } + + /** + * Source: + * ``` + * import java.util.List; + * + * public class Dollar$InClass { + * void dollarIn$Method() {} + * int $inField; + * } + * + * ``` + */ + @Test fun `testDollar$InClass`() { + doTest(""" + ${'$'}inField + Code + Dollar${'$'}InClass + I + LineNumberTable + LocalVariableTable + Object + SourceFile + dollarIn${'$'}Method + java + lang + this + """.trimIndent()) + } + + private fun doTest(text: String) { + val filePath = constantPoolPath.resolve(getTestName(false) + ".class") + val bytes = filePath.readBytes() + val ids = extractIdsFromClassFile(bytes) + Assertions.assertEquals(text.trim(), ids.joinToString("\n").trim()) + } +} + +internal fun extractIdsFromClassFile(bytes: ByteArray): List { + val parts = ArrayList() + ConstantPoolParser(DataInputStream(ByteArrayInputStream(bytes))) { + val tmp = ArrayList() + if (collectJvmIdentifiers(it) { sequence: CharSequence, start: Int, end: Int -> tmp.add(sequence.subSequence(start, end)) }) { + parts.addAll(tmp) + } + }.parse() + val ids = parts.map { it.toString() }.toSet().sorted() + return ids +} \ No newline at end of file diff --git a/java/java-tests/testSrc/com/intellij/psi/impl/JvmIdentifierUtilTest.java b/java/java-tests/testSrc/com/intellij/psi/impl/JvmIdentifierUtilTest.java new file mode 100644 index 000000000000..2be8590a1e91 --- /dev/null +++ b/java/java-tests/testSrc/com/intellij/psi/impl/JvmIdentifierUtilTest.java @@ -0,0 +1,45 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.psi.impl; + +import com.intellij.psi.impl.cache.impl.idCache.JvmIdentifierUtil; +import com.intellij.util.containers.ContainerUtil; +import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +public class JvmIdentifierUtilTest { + @Test + void testIdentifierExtraction() { + Assertions.assertEquals(List.of("foo"), getJvmIdParts("foo")); + Assertions.assertEquals(List.of("foo", "bar"), getJvmIdParts("foo/bar")); + Assertions.assertEquals(List.of("a", "bb", "ccc"), getJvmIdParts("a/bb/ccc")); + Assertions.assertEquals(List.of("aaa", "bb", "c"), getJvmIdParts("aaa/bb/c")); + Assertions.assertEquals(List.of("java", "lang", "String"), getJvmIdParts("[[Ljava/lang/String;")); + Assertions.assertEquals(List.of("java", "lang", "Comparable", "java", "lang", "String", "java", "util", "List", "java", "lang", "Integer", "java", "lang", "Object"), getJvmIdParts(";:Ljava/util/List;>Ljava/lang/Object;")); + Assertions.assertEquals(List.of( + "java", "util", "Map", "java", "lang", "String", "java", "util", "List", "java", "util", "List", "java", "util", "Comparator", "java", "lang", "String" + ), getJvmIdParts("Ljava/util/Map<[Ljava/lang/String;Ljava/util/List;>;>;>;")); + Assertions.assertEquals(List.of("foo$bar"), getJvmIdParts("foo$bar")); + Assertions.assertNull(getJvmIdParts("1")); + Assertions.assertNull(getJvmIdParts("1/bar")); + Assertions.assertNull(getJvmIdParts("/")); + Assertions.assertNull(getJvmIdParts("with/spa ce")); + Assertions.assertNull(getJvmIdParts("asd*")); + Assertions.assertNull(getJvmIdParts("My sentence with spaces")); + Assertions.assertNull(getJvmIdParts("")); + Assertions.assertNull(getJvmIdParts("foo/")); + Assertions.assertNull(getJvmIdParts("/foo")); + } + + @Nullable + List getJvmIdParts(String str) { + ArrayList parts = new ArrayList<>(); + if (JvmIdentifierUtil.collectJvmIdentifiers(str, (sequence1, start, end) -> parts.add(sequence1.subSequence(start, end)))) { + return ContainerUtil.map(parts, sequence -> sequence.toString()); + } + return null; + } +} diff --git a/platform/indexing-api/api-dump-unreviewed.txt b/platform/indexing-api/api-dump-unreviewed.txt index 67f7db5c0073..8c3f9563600f 100644 --- a/platform/indexing-api/api-dump-unreviewed.txt +++ b/platform/indexing-api/api-dump-unreviewed.txt @@ -240,6 +240,9 @@ c:com.intellij.lang.findUsages.LanguageFindUsages - a:getElement():com.intellij.psi.PsiElement - a:getOffsetInElement():I - s:of(com.intellij.psi.PsiElement,I):com.intellij.model.search.TextOccurrence +com.intellij.platform.indexing.BinaryFileSourceProvider +- sf:EP:com.intellij.openapi.extensions.ExtensionPointName +- a:findSourceFile(com.intellij.psi.PsiBinaryFile):com.intellij.psi.PsiFile com.intellij.psi.impl.cache.CacheManager - a:getFilesWithWord(java.lang.String,S,com.intellij.psi.search.GlobalSearchScope,Z):com.intellij.psi.PsiFile[] - s:getInstance(com.intellij.openapi.project.Project):com.intellij.psi.impl.cache.CacheManager diff --git a/platform/indexing-api/resources/META-INF/Indexing.xml b/platform/indexing-api/resources/META-INF/Indexing.xml index 3f50a794316a..d2c56e1f3e5a 100644 --- a/platform/indexing-api/resources/META-INF/Indexing.xml +++ b/platform/indexing-api/resources/META-INF/Indexing.xml @@ -22,6 +22,7 @@ + diff --git a/platform/indexing-api/src/com/intellij/platform/indexing/BinaryFileSourceProvider.java b/platform/indexing-api/src/com/intellij/platform/indexing/BinaryFileSourceProvider.java new file mode 100644 index 000000000000..cfbff6bba697 --- /dev/null +++ b/platform/indexing-api/src/com/intellij/platform/indexing/BinaryFileSourceProvider.java @@ -0,0 +1,27 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.platform.indexing; + +import com.intellij.openapi.extensions.ExtensionPointName; +import com.intellij.psi.PsiBinaryFile; +import com.intellij.psi.PsiFile; +import com.intellij.util.concurrency.annotations.RequiresReadLock; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * In the situation when the identifiers are provided by the binary file (implemented {@link com.intellij.psi.impl.cache.impl.id.IdIndexer} + * for binary file) which corresponds to some source file, this extension is used to find the original source file during the Find Usages. + */ +public interface BinaryFileSourceProvider { + ExtensionPointName EP = new ExtensionPointName<>("com.intellij.binaryFileSourceProvider"); + + /** + * Finds the original source file corresponding to the given binary file. + * + * @param file the binary file for which to find the source file + * @return the original source file, or {@code null} if not found + */ + @Nullable + @RequiresReadLock + PsiFile findSourceFile(@NotNull PsiBinaryFile file); +} diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/BaseFilterLexerUtil.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/BaseFilterLexerUtil.java index f03415d7e791..2186ee356e29 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/BaseFilterLexerUtil.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/BaseFilterLexerUtil.java @@ -12,6 +12,7 @@ import com.intellij.psi.impl.cache.impl.todo.TodoIndexers; import com.intellij.psi.search.IndexPattern; import com.intellij.testFramework.LightVirtualFile; import com.intellij.util.indexing.FileContent; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import java.util.Collections; @@ -48,7 +49,8 @@ public final class BaseFilterLexerUtil { return todoMap != null ? todoMap : Collections.emptyMap(); } - private static void scanContentWithCheckCanceled(@NotNull FileContent content, @NotNull Lexer filterLexer) { + @ApiStatus.Internal + public static void scanContentWithCheckCanceled(@NotNull FileContent content, @NotNull Lexer filterLexer) { filterLexer.start(content.getContentAsText()); int tokenIdx = 0; while (filterLexer.getTokenType() != null) { diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdDataConsumer.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdDataConsumer.java index d97193c5a22e..655302e1f921 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdDataConsumer.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/id/IdDataConsumer.java @@ -104,7 +104,6 @@ public final class IdDataConsumer { myOccurrences.forEach((hash, value) -> consumer.accept(new IdIndexEntry(hash), value)); } }; - public @NotNull Map getResult() { return myResult; } diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java b/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java index 02475c668102..16226a59cfb7 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java @@ -31,6 +31,7 @@ import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.util.text.TrigramBuilder; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.platform.indexing.BinaryFileSourceProvider; import com.intellij.psi.*; import com.intellij.psi.impl.PsiManagerEx; import com.intellij.psi.impl.cache.CacheManager; @@ -370,7 +371,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { try { progress.setText(IndexingBundle.message("psi.scanning.files.progress")); - Processor localProcessor = localProcessor(searcher, processor); + Processor localProcessor = localProcessor(searcher, processor); // Lists of files to search in this order. // First, there are lists with higher probability of hits (e.g., files with `containerName` or files near the target) @@ -449,6 +450,9 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { } /** + * NOTE: candidateFile and file might be actually 2 different files (e.g. we may find class file in java, but the PsiFile + * will be from mirror source class) + * * @param files to scan for references in this pass. * @param totalSize the number of files to scan in both passes. Can be different from {@code files.size()} in case of * two-pass scan, where we first scan files containing container name and then all the rest files. @@ -459,7 +463,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { int totalSize, int alreadyProcessedFiles, @NotNull ProgressIndicator progress, - @NotNull Processor localProcessor) { + @NotNull Processor localProcessor) { return myManager.runInBatchFilesMode(() -> { AtomicInteger counter = new AtomicInteger(alreadyProcessedFiles); AtomicBoolean stopped = new AtomicBoolean(false); @@ -488,6 +492,15 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { }); } + /** + * NOTE: candidateFile and file might be actually two different files (e.g. we may find class file in java, but the PsiFile + * will be from mirror source class) + */ + record CandidateFileInfo( + VirtualFile candidateFile, + PsiFile file + ) { } + // Tries to run {@code localProcessor} for each file in {@code files} concurrently on ForkJoinPool. // When encounters write action request, stops all threads, waits for write action to finish and re-starts all threads again, // trying to finish the unprocessed files (i.e. those for which {@code localProcessor} hasn't been called yet). @@ -590,7 +603,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { private void processVirtualFile(@NotNull VirtualFile vfile, @NotNull AtomicBoolean stopped, - @NotNull Processor localProcessor) throws ApplicationUtil.CannotRunReadActionException { + @NotNull Processor localProcessor) throws ApplicationUtil.CannotRunReadActionException { // try to pre-cache virtual file content outside read action to avoid stalling EDT if (!vfile.isDirectory() && !vfile.getFileType().isBinary()) { try { @@ -608,6 +621,13 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { file = ((BackFileViewProvider)file.getViewProvider()).getFrontPsiFile(); } + if (file instanceof PsiBinaryFile binaryFile) { + PsiFile originalFile = findOriginalFile(binaryFile); + if (originalFile != null) { + file = originalFile; + } + } + if (file != null && !(file instanceof PsiBinaryFile)) { Project project = myManager.getProject(); if (project.isDisposed()) throw new ProcessCanceledException(); @@ -627,7 +647,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { continue; } - if (!localProcessor.process(psiRoot)) { + if (!localProcessor.process(new CandidateFileInfo(vfile, psiRoot))) { stopped.set(true); break; } @@ -638,6 +658,16 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { } } + @Nullable + private static PsiFile findOriginalFile(@NotNull PsiBinaryFile file) { + List providers = BinaryFileSourceProvider.EP.getExtensionList(); + for (BinaryFileSourceProvider provider : providers) { + PsiFile originalFile = provider.findSourceFile(file); + if (originalFile != null) return originalFile; + } + return null; + } + private void getFilesWithText(@NotNull GlobalSearchScope scope, short searchContext, boolean caseSensitively, @@ -793,7 +823,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { Map> globals = new HashMap<>(); List> customs = new ArrayList<>(); Set locals = new LinkedHashSet<>(); - Map> localProcessors = new HashMap<>(); + Map> localProcessors = new HashMap<>(); distributePrimitives(collectors, locals, globals, customs, localProcessors); if (!processGlobalRequestsOptimized(globals, progress, localProcessors)) { return false; @@ -834,10 +864,11 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { CHANGED, } - static @NotNull Processor localProcessor(@NotNull StringSearcher searcher, @NotNull BulkOccurrenceProcessor processor) { + static @NotNull Processor localProcessor(@NotNull StringSearcher searcher, @NotNull BulkOccurrenceProcessor processor) { return new ReadActionProcessor<>() { @Override - public boolean processInReadAction(PsiElement scopeElement) { + public boolean processInReadAction(CandidateFileInfo candidateFileInfo) { + PsiElement scopeElement = candidateFileInfo.file(); if (scopeElement instanceof PsiCompiledElement) { // can't scan text of the element return true; @@ -856,7 +887,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { private boolean processGlobalRequestsOptimized(@NotNull Map> singles, @NotNull ProgressIndicator progress, - @NotNull Map> localProcessors) { + @NotNull Map> localProcessors) { if (singles.isEmpty()) { return true; } @@ -874,7 +905,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { boolean processGlobalRequests(@NotNull Map> singles, @NotNull ProgressIndicator progress, - @NotNull Map> localProcessors) { + @NotNull Map> localProcessors) { progress.pushState(); progress.setText(IndexingBundle.message("psi.scanning.files.progress")); boolean result; @@ -926,22 +957,22 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { return result; } - private boolean processCandidates(@NotNull Map> localProcessors, + private boolean processCandidates(@NotNull Map> localProcessors, @NotNull Map> candidateFiles, @NotNull ProgressIndicator progress, int totalSize, int alreadyProcessedFiles) { List files = new ArrayList<>(candidateFiles.keySet()); - return processPsiFileRoots(files, totalSize, alreadyProcessedFiles, progress, psiRoot -> { - VirtualFile vfile = psiRoot.getVirtualFile(); + return processPsiFileRoots(files, totalSize, alreadyProcessedFiles, progress, candidateInfo -> { + VirtualFile vfile = candidateInfo.candidateFile(); if (vfile instanceof BackedVirtualFile) { vfile = ((BackedVirtualFile)vfile).getOriginFile(); } for (T singleRequest : candidateFiles.get(vfile)) { ProgressManager.checkCanceled(); - Processor localProcessor = localProcessors.get(singleRequest); - if (!localProcessor.process(psiRoot)) { + Processor localProcessor = localProcessors.get(singleRequest); + if (!localProcessor.process(candidateInfo)) { return false; } } @@ -1090,7 +1121,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { @NotNull Set locals, @NotNull Map> globals, @NotNull List> customs, - @NotNull Map> localProcessors) { + @NotNull Map> localProcessors) { for (Map.Entry> entry : collectors.entrySet()) { ProgressManager.checkCanceled(); Processor processor = entry.getValue(); @@ -1120,9 +1151,9 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { StringSearcher searcher = new StringSearcher(primitive.word, primitive.caseSensitive, true, false); BulkOccurrenceProcessor adapted = adaptProcessor(primitive, singleRequest.refProcessor); - Processor localProcessor = localProcessor(searcher, adapted); + Processor localProcessor = localProcessor(searcher, adapted); - Processor old = localProcessors.put(singleRequest, localProcessor); + Processor old = localProcessors.put(singleRequest, localProcessor); assert old == null : old + ";" + localProcessor +"; singleRequest="+singleRequest; } } diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/search/helper.kt b/platform/indexing-impl/src/com/intellij/psi/impl/search/helper.kt index 3a1662c328c7..f720e30826f9 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/search/helper.kt +++ b/platform/indexing-impl/src/com/intellij/psi/impl/search/helper.kt @@ -9,7 +9,7 @@ import com.intellij.openapi.project.DumbService import com.intellij.openapi.project.Project import com.intellij.openapi.util.ClassExtension import com.intellij.openapi.util.Computable -import com.intellij.psi.PsiElement +import com.intellij.psi.impl.search.PsiSearchHelperImpl.CandidateFileInfo import com.intellij.psi.search.LocalSearchScope import com.intellij.psi.search.PsiSearchHelper import com.intellij.psi.search.SearchSession @@ -225,8 +225,8 @@ private class Layer( return myHelper.processGlobalRequests(globalsIds, progress, scopeProcessors(globals)) } - private fun scopeProcessors(globals: Collection): Map> { - val result = HashMap>() + private fun scopeProcessors(globals: Collection): Map> { + val result = HashMap>() for (requestAndProcessors: RequestAndProcessors in globals) { progress.checkCanceled() result[requestAndProcessors.request] = scopeProcessor(requestAndProcessors) @@ -234,7 +234,7 @@ private class Layer( return result } - private fun scopeProcessor(requestAndProcessors: RequestAndProcessors): Processor { + private fun scopeProcessor(requestAndProcessors: RequestAndProcessors): Processor { val (request: WordRequestInfo, processors: RequestProcessors) = requestAndProcessors val searcher = StringSearcher(request.word, request.isCaseSensitive, true, false) val adapted = MyBulkOccurrenceProcessor(project, processors) diff --git a/platform/util/api-dump-unreviewed.txt b/platform/util/api-dump-unreviewed.txt index 7bc094c876e1..9bd8d90c88a3 100644 --- a/platform/util/api-dump-unreviewed.txt +++ b/platform/util/api-dump-unreviewed.txt @@ -1973,6 +1973,7 @@ c:com.intellij.openapi.util.text.StringUtil - s:isEmptyOrSpaces(java.lang.String):Z - s:isEscapedBackslash(java.lang.CharSequence,I,I):Z - s:isHexDigit(C):Z +- s:isJavaIdentifier(java.lang.CharSequence,I,I):Z - s:isJavaIdentifier(java.lang.String):Z - s:isJavaIdentifierPart(C):Z - s:isJavaIdentifierStart(C):Z diff --git a/platform/util/src/com/intellij/openapi/util/text/StringUtil.java b/platform/util/src/com/intellij/openapi/util/text/StringUtil.java index 19b00df554f7..625ddbc9544d 100644 --- a/platform/util/src/com/intellij/openapi/util/text/StringUtil.java +++ b/platform/util/src/com/intellij/openapi/util/text/StringUtil.java @@ -11,6 +11,7 @@ import com.intellij.util.*; import com.intellij.util.text.CharArrayUtil; import com.intellij.util.text.CharSequenceSubSequence; import com.intellij.util.text.MergingCharSequence; +import kotlin.jvm.internal.Ref; import org.jetbrains.annotations.*; import org.jetbrains.annotations.ApiStatus.NonExtendable; @@ -2275,6 +2276,9 @@ public class StringUtil { return cp >= '0' && cp <= '9' || cp >= 'a' && cp <= 'z' || cp >= 'A' && cp <= 'Z' || Character.isJavaIdentifierPart(cp); } + /** + * @return true iff the string is a valid java identifier (according to JLS 3.8) + */ @Contract(pure = true) public static boolean isJavaIdentifier(@NotNull String text) { int len = text.length(); @@ -2291,6 +2295,58 @@ public class StringUtil { return true; } + /** + * Checks if the char sequence slice is a valid java identifier (according to JLS 3.8) + * @return true exactly iff {@code isJavaIdentifier(str.subSequence(startIncl, endExcl)) == true } + */ + public static boolean isJavaIdentifier(@NotNull CharSequence str, int startIncl, int endExcl) { + if (str.length() == 0) return false; + Ref.BooleanRef isFirst = new Ref.BooleanRef(); + isFirst.element = true; + return processCodePoints(str, startIncl, endExcl, codePoint -> { + if (isFirst.element) { + isFirst.element = false; + if (!isJavaIdentifierStart(codePoint)) { + return false; + } + } else { + if (!isJavaIdentifierPart(codePoint)) { + return false; + } + } + return true; + }); + } + + private interface CodePointProcessor { + /** + * @return true iff should continue + */ + boolean acceptCodePoint(int codePoint); + } + + + private static boolean processCodePoints(@NotNull CharSequence str, int startIncl, int endExcl, CodePointProcessor processor) { + int i = startIncl; + while (i < endExcl) { + char c1 = str.charAt(i++); + if (!Character.isHighSurrogate(c1) || i >= endExcl) { + if (!processor.acceptCodePoint(c1)) return false; + } + else { + char c2 = str.charAt(i); + if (Character.isLowSurrogate(c2)) { + i++; + if (!processor.acceptCodePoint(Character.toCodePoint(c1, c2))) return false; + } + else { + if (!processor.acceptCodePoint(c2)) return false; + } + } + } + return true; + } + /** * Escape property name or key in property file. Unicode characters are escaped as well. * diff --git a/platform/util/testSrc/com/intellij/util/text/StringUtilTest.java b/platform/util/testSrc/com/intellij/util/text/StringUtilTest.java index b051c93ba4b9..27ab09d678cb 100644 --- a/platform/util/testSrc/com/intellij/util/text/StringUtilTest.java +++ b/platform/util/testSrc/com/intellij/util/text/StringUtilTest.java @@ -943,6 +943,23 @@ public class StringUtilTest { assertTrue(StringUtil.isJavaIdentifier("\u03B1A")); } + @SuppressWarnings("UnnecessaryUnicodeEscape") + @Test + public void testCharSequenceSliceIsJavaIdentifier() { + assertFalse(StringUtil.isJavaIdentifier("", 0, 0)); + assertTrue(StringUtil.isJavaIdentifier("x", 0, 1)); + assertFalse(StringUtil.isJavaIdentifier("0", 0, 1)); + assertFalse(StringUtil.isJavaIdentifier("0x", 0, 2)); + assertTrue(StringUtil.isJavaIdentifier("foo$bar", 0, 7)); + assertTrue(StringUtil.isJavaIdentifier("x0", 0, 2)); + assertTrue(StringUtil.isJavaIdentifier("\uD835\uDEFCA", 0, 3)); + assertTrue(StringUtil.isJavaIdentifier("A\uD835\uDEFC", 0, 3)); + assertTrue(StringUtil.isJavaIdentifier("\u03B1A", 0, 2)); + assertTrue(StringUtil.isJavaIdentifier("###\u03B1A", 3, 5)); + assertTrue(StringUtil.isJavaIdentifier("\u03B1A###", 0, 2)); + assertTrue(StringUtil.isJavaIdentifier("###\u03B1A###", 3, 5)); + } + @Test public void testSplit() { String spaceSeparator = " ";