MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

rename TokenizedText into TokenList, as it's more about tokens than about text

Browse Source 
GitOrigin-RevId: 9f3a73f62cde0649723f6d7f0c4083241d45db20
This commit is contained in:
Peter Gromov
2020-06-15 16:27:41 +03:00
committed by intellij-monorepo-bot
parent cb8d732078
commit b4a7ed1da3
6 changed files with 24 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/java-indexing-impl/src/com/intellij/psi/impl/java/JavaBinaryPlusExpressionIndex.java
+2 -2
View File
@@ -3,7 +3,7 @@ package com.intellij.psi.impl.java;
import com.intellij.ide.highlighter.JavaFileType;
import com.intellij.lang.java.parser.JavaParserUtil;
import com.intellij.lexer.TokenizedText;
import com.intellij.lexer.TokenList;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.impl.source.JavaFileElementType;
import com.intellij.psi.impl.source.tree.ElementType;
@@ -37,7 +37,7 @@ public class JavaBinaryPlusExpressionIndex extends FileBasedIndexExtension<Boole
@Override
public DataIndexer<Boolean, PlusOffsets, FileContent> getIndexer() {
return inputData -> {
TokenizedText tokens = JavaParserUtil.obtainTokens(inputData.getPsiFile());
TokenList tokens = JavaParserUtil.obtainTokens(inputData.getPsiFile());
TIntArrayList result = new TIntArrayList();
for (int i = 0; i < tokens.getTokenCount(); i++) {
java/java-indexing-impl/src/com/intellij/psi/impl/search/JavaNullMethodArgumentIndex.java
+4 -4
View File
@@ -4,7 +4,7 @@ package com.intellij.psi.impl.search;
import com.intellij.ide.highlighter.JavaFileType;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lang.java.parser.JavaParserUtil;
import com.intellij.lexer.TokenizedText;
import com.intellij.lexer.TokenList;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.tree.IElementType;
@@ -46,7 +46,7 @@ public final class JavaNullMethodArgumentIndex extends ScalarIndexExtension<Java
Map<MethodCallData, Void> result = new THashMap<>();
TokenizedText tokens = JavaParserUtil.obtainTokens(inputData.getPsiFile());
TokenList tokens = JavaParserUtil.obtainTokens(inputData.getPsiFile());
for (int i = 0; i < tokens.getTokenCount(); i++) {
if (tokens.hasType(i, NULL_KEYWORD)) {
MethodCallData data = findCallData(tokens, i);
@@ -60,7 +60,7 @@ public final class JavaNullMethodArgumentIndex extends ScalarIndexExtension<Java
}
@Nullable
private static MethodCallData findCallData(TokenizedText tokens, int nullIndex) {
private static MethodCallData findCallData(TokenList tokens, int nullIndex) {
if (!tokens.hasType(tokens.forwardWhile(nullIndex + 1, JavaParserUtil.WS_COMMENTS), RPARENTH, COMMA)) return null;
int i = tokens.backWhile(nullIndex - 1, JavaParserUtil.WS_COMMENTS);
@@ -87,7 +87,7 @@ public final class JavaNullMethodArgumentIndex extends ScalarIndexExtension<Java
}
@Nullable
private static String findMethodName(TokenizedText tokens, int lparenth) {
private static String findMethodName(TokenList tokens, int lparenth) {
int i = tokens.backWhile(lparenth - 1, JavaParserUtil.WS_COMMENTS);
if (tokens.hasType(i, GT)) {
i = tokens.backWhile(tokens.backWithBraceMatching(i, LT, GT), JavaParserUtil.WS_COMMENTS);
java/java-psi-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java
+2 -2
View File
@@ -8,7 +8,7 @@ import com.intellij.lang.impl.TokenSequence;
import com.intellij.lang.java.JavaLanguage;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.lexer.TokenizedText;
import com.intellij.lexer.TokenList;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Condition;
import com.intellij.openapi.util.Key;
@@ -40,7 +40,7 @@ public class JavaParserUtil {
private static final Key<Boolean> DEEP_PARSE_BLOCKS_IN_STATEMENTS = Key.create("JavaParserUtil.ParserExtender");
@NotNull
public static TokenizedText obtainTokens(@NotNull PsiFile file) {
public static TokenList obtainTokens(@NotNull PsiFile file) {
return CachedValuesManager.getCachedValue(file, () ->
CachedValueProvider.Result.create(
TokenSequence.performLexing(file.getViewProvider().getContents(), JavaParserDefinition.createLexer(PsiUtil.getLanguageLevel(file))),
java/java-psi-impl/src/com/intellij/psi/impl/source/JavaLightStubBuilder.java
+2 -2
View File
@@ -22,7 +22,7 @@ import com.intellij.lang.LighterLazyParseableNode;
import com.intellij.lang.impl.TokenSequence;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.lexer.TokenizedText;
import com.intellij.lexer.TokenList;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.PsiFile;
import com.intellij.psi.PsiJavaFile;
@@ -70,7 +70,7 @@ public class JavaLightStubBuilder extends LightStubBuilder {
CodeBlockVisitor visitor = new CodeBlockVisitor();
if (TreeUtil.isCollapsedChameleon(node)) {
Lexer lexer = JavaParserDefinition.createLexer(PsiUtil.getLanguageLevel(node.getPsi()));
TokenizedText tokens = TokenSequence.performLexing(node.getChars(), lexer);
TokenList tokens = TokenSequence.performLexing(node.getChars(), lexer);
for (int i = 0; i < tokens.getTokenCount(); i++) {
visitor.visit(tokens.getTokenType(i));
}
platform/core-api/src/com/intellij/lexer/TokenizedText.java → platform/core-api/src/com/intellij/lexer/TokenList.java
+10 -10
View File
@@ -14,7 +14,7 @@ import org.jetbrains.annotations.Nullable;
* It allows clients to inspect all tokens at once and easily move back and forward to implement some simple lexer-based checks.
*/
@ApiStatus.Experimental
public interface TokenizedText {
public interface TokenList {
/**
* @return the number of tokens inside
@@ -25,7 +25,7 @@ public interface TokenizedText {
* @return the full text that was split into the tokens represented here
*/
@NotNull
CharSequence getText();
CharSequence getTokenizedText();
/**
* @return the start offset of the token with the given index
@@ -47,7 +47,7 @@ public interface TokenizedText {
*/
default CharSequence getTokenText(int index) {
if (index < 0 || index >= getTokenCount()) return null;
return getText().subSequence(getTokenStart(index), getTokenEnd(index));
return getTokenizedText().subSequence(getTokenStart(index), getTokenEnd(index));
}
/**
@@ -130,23 +130,23 @@ public interface TokenizedText {
}
/**
* A simple lexer over {@link TokenizedText}.
* A simple lexer over {@link TokenList}.
*/
class WrappingLexer extends LexerBase {
private final TokenizedText myTokens;
private final TokenList myTokens;
private int myIndex;
WrappingLexer(TokenizedText tokens) {
WrappingLexer(TokenList tokens) {
this.myTokens = tokens;
}
public TokenizedText getTokens() {
public TokenList getTokens() {
return myTokens;
}
@Override
public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
assert Comparing.equal(buffer, myTokens.getText());
assert Comparing.equal(buffer, myTokens.getTokenizedText());
assert startOffset == 0;
assert endOffset == buffer.length();
assert initialState == 0;
@@ -180,12 +180,12 @@ public interface TokenizedText {
@Override
public @NotNull CharSequence getBufferSequence() {
return myTokens.getText();
return myTokens.getTokenizedText();
}
@Override
public int getBufferEnd() {
return myTokens.getText().length();
return myTokens.getTokenizedText().length();
}
}
platform/core-impl/src/com/intellij/lang/impl/TokenSequence.java
+4 -4
View File
@@ -16,7 +16,7 @@
package com.intellij.lang.impl;
import com.intellij.lexer.Lexer;
import com.intellij.lexer.TokenizedText;
import com.intellij.lexer.TokenList;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressIndicatorProvider;
import com.intellij.openapi.util.Comparing;
@@ -27,7 +27,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@ApiStatus.Experimental
public class TokenSequence implements TokenizedText {
public class TokenSequence implements TokenList {
private static final Logger LOG = Logger.getInstance(TokenSequence.class);
private final CharSequence myText;
@@ -57,7 +57,7 @@ public class TokenSequence implements TokenizedText {
@NotNull
public static TokenSequence performLexing(@NotNull CharSequence text, @NotNull Lexer lexer) {
if (lexer instanceof WrappingLexer) {
TokenizedText existing = ((WrappingLexer)lexer).getTokens();
TokenList existing = ((WrappingLexer)lexer).getTokens();
if (existing instanceof TokenSequence && Comparing.equal(text, ((TokenSequence)existing).myText)) {
// prevent clients like PsiBuilder from modifying shared token types
return new TokenSequence(((TokenSequence)existing).lexStarts,
@@ -90,7 +90,7 @@ public class TokenSequence implements TokenizedText {
}
@Override
public @NotNull CharSequence getText() {
public @NotNull CharSequence getTokenizedText() {
return myText;
}