cleanup ITokenTypeRemapper

GitOrigin-RevId: 8c2fd80bbef25892ac0c2a6c593c4601124fdf23
This commit is contained in:
Max Medvedev
2024-12-22 19:08:48 +00:00
committed by intellij-monorepo-bot
parent 841d9340a3
commit e26bd43692
3 changed files with 15 additions and 5 deletions
@@ -17,17 +17,21 @@
package com.intellij.lang;
import com.intellij.psi.tree.IElementType;
import org.jetbrains.annotations.NotNull;
@FunctionalInterface
public interface ITokenTypeRemapper {
/**
* An external hook to see and alter token types reported by lexer.
* A lexer might take a delegate implementing this interface.
* @param source type of an element as lexer understood it.
* @param source type of element as lexer understood it.
* @param start start index of lexeme in text (as lexer.getTokenStart() would return).
* @param end end index of lexeme in text (as lexer.getTokenEnd() would return).
* @param text text being parsed.
* @return altered (or not) element type.
**/
IElementType filter(final IElementType source, final int start, final int end, final CharSequence text);
@NotNull IElementType filter(@NotNull IElementType source,
int start,
int end,
@NotNull CharSequence text);
}
@@ -15,6 +15,7 @@ import com.jetbrains.python.psi.FutureFeature;
import com.jetbrains.python.psi.LanguageLevel;
import com.jetbrains.python.psi.PyElementType;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.EnumSet;
@@ -1080,11 +1081,15 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
}
@Override
public IElementType filter(final IElementType source, final int start, final int end, final CharSequence text) {
public @NotNull IElementType filter(@NotNull IElementType source, int start, int end, @NotNull CharSequence text) {
return filter(source, start, end, text, true);
}
protected IElementType filter(final IElementType source, final int start, final int end, final CharSequence text, boolean checkLanguageLevel) {
protected @NotNull IElementType filter(@NotNull IElementType source,
int start,
int end,
@NotNull CharSequence text,
boolean checkLanguageLevel) {
if (source == PyTokenTypes.IDENTIFIER && isWordAtPosition(text, start, end, TOK_AS)) {
return PyTokenTypes.AS_KEYWORD;
}
@@ -23,6 +23,7 @@ import com.jetbrains.python.parsing.ExpressionParsing;
import com.jetbrains.python.parsing.ParsingContext;
import com.jetbrains.python.parsing.StatementParsing;
import com.jetbrains.python.psi.LanguageLevel;
import org.jetbrains.annotations.NotNull;
/**
* User : ktisha
@@ -70,7 +71,7 @@ public class PyDocstringParsingContext extends ParsingContext {
}
@Override
public IElementType filter(IElementType source, int start, int end, CharSequence text) {
public @NotNull IElementType filter(@NotNull IElementType source, int start, int end, @NotNull CharSequence text) {
if (source == PyTokenTypes.DOT && CharArrayUtil.regionMatches(text, start, end, "..."))
return PyDocstringTokenTypes.DOTS;
if (source == PyTokenTypes.GTGT && CharArrayUtil.regionMatches(text, start, end, ">>>"))