MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

- do not scan words when token is expected to have no words (e.g. whitespace)

Browse Source 
- skip filterlexer creation when indexing
This commit is contained in:
Maxim.Mossienko
2012-02-07 19:25:04 +04:00
parent e654e23b5a
commit 9e43a15489
9 changed files with 77 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaFilterLexer.java
Vendored
+22 -3
View File
@@ -22,18 +22,37 @@ import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
import com.intellij.psi.impl.source.tree.ElementType;
import com.intellij.psi.search.UsageSearchContext;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
/**
* @author ven
*/
public class JavaFilterLexer extends BaseFilterLexer {
private static final TokenSet ourSkipWordsScanSet = TokenSet.orSet(
TokenSet.create(
JavaTokenType.WHITE_SPACE,
JavaTokenType.LPARENTH,
JavaTokenType.RPARENTH,
JavaTokenType.LBRACE,
JavaTokenType.RBRACE,
JavaTokenType.LBRACKET,
JavaTokenType.RBRACKET,
JavaTokenType.SEMICOLON,
JavaTokenType.COMMA,
JavaTokenType.DOT,
JavaTokenType.ELLIPSIS,
JavaTokenType.AT
),
JavaTokenType.OPERATION_BIT_SET
);
public JavaFilterLexer(final Lexer originalLexer, final OccurrenceConsumer table) {
super(originalLexer, table);
}
@Override
public void advance() {
final IElementType tokenType = getDelegate().getTokenType();
final IElementType tokenType = myDelegate.getTokenType();
if (tokenType == JavaTokenType.IDENTIFIER
|| tokenType == JavaTokenType.LONG_LITERAL
@@ -48,10 +67,10 @@ public class JavaFilterLexer extends BaseFilterLexer {
scanWordsInToken(UsageSearchContext.IN_COMMENTS, false, false);
advanceTodoItemCountsInToken();
}
else {
else if (!ourSkipWordsScanSet.contains(tokenType)) {
scanWordsInToken(UsageSearchContext.IN_PLAIN_TEXT, false, false);
}
getDelegate().advance();
myDelegate.advance();
}
}
java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexer.java
Vendored
+1 -4
View File
@@ -15,13 +15,11 @@
*/
package com.intellij.psi.impl.cache.impl.idCache;
import com.intellij.lexer.FilterLexer;
import com.intellij.lexer.JavaLexer;
import com.intellij.lexer.Lexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer;
import com.intellij.psi.impl.source.tree.StdTokenSets;
public class JavaIdIndexer extends LexerBasedIdIndexer {
@Override
@@ -31,7 +29,6 @@ public class JavaIdIndexer extends LexerBasedIdIndexer {
public static Lexer createIndexingLexer(OccurrenceConsumer consumer) {
final JavaLexer javaLexer = new JavaLexer(LanguageLevel.JDK_1_3);
final JavaFilterLexer filterLexer = new JavaFilterLexer(javaLexer, consumer);
return new FilterLexer(filterLexer, new FilterLexer.SetFilter(StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET));
return new JavaFilterLexer(javaLexer, consumer);
}
}
java/java-psi-impl/src/com/intellij/lexer/JavaLexer.java
+3 -8
View File
@@ -196,7 +196,7 @@ public class JavaLexer extends LexerBase {
@Override
public final IElementType getTokenType() {
locateToken();
if (myTokenType == null) _locateToken();
return myTokenType;
}
@@ -208,22 +208,17 @@ public class JavaLexer extends LexerBase {
@Override
public final int getTokenEnd() {
locateToken();
if (myTokenType == null) _locateToken();
return myTokenEndOffset;
}
@Override
public final void advance() {
locateToken();
if (myTokenType == null) _locateToken();
myTokenType = null;
}
protected final void locateToken() {
if (myTokenType != null) return;
_locateToken();
}
private void _locateToken() {
if (myTokenEndOffset == myBufferEndOffset) {
myTokenType = null;
platform/core-api/src/com/intellij/lexer/DelegateLexer.java
+1 -1
View File
@@ -23,7 +23,7 @@ import com.intellij.psi.tree.IElementType;
import org.jetbrains.annotations.Nullable;
public class DelegateLexer extends LexerBase {
private final Lexer myDelegate;
protected final Lexer myDelegate;
public DelegateLexer(Lexer delegate) {
myDelegate = delegate;
platform/lang-api/src/com/intellij/lang/cacheBuilder/DefaultWordsScanner.java
+20 -5
View File
@@ -19,6 +19,7 @@ import com.intellij.lexer.Lexer;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NotNull;
/**
* The default implementation of a words scanner based on a custom language lexer.
@@ -31,6 +32,7 @@ public class DefaultWordsScanner implements WordsScanner {
private final TokenSet myIdentifierTokenSet;
private final TokenSet myCommentTokenSet;
private final TokenSet myLiteralTokenSet;
private final TokenSet mySkipCodeContextTokenSet;
private boolean myMayHaveFileRefsInLiterals;
/**
@@ -43,18 +45,33 @@ public class DefaultWordsScanner implements WordsScanner {
*/
public DefaultWordsScanner(final Lexer lexer, final TokenSet identifierTokenSet, final TokenSet commentTokenSet,
final TokenSet literalTokenSet) {
this(lexer, identifierTokenSet, commentTokenSet, literalTokenSet, TokenSet.EMPTY);
}
/**
* Creates a new instance of the words scanner.
*
* @param lexer the lexer used for breaking the text into tokens.
* @param identifierTokenSet the set of token types which represent identifiers.
* @param commentTokenSet the set of token types which represent comments.
* @param literalTokenSet the set of token types which represent literals.
* @param SkipCodeContextTokenSet the set of token types which should not be considered as code context.
*/
public DefaultWordsScanner(final Lexer lexer, final TokenSet identifierTokenSet, final TokenSet commentTokenSet,
final TokenSet literalTokenSet, @NotNull TokenSet skipCodeContextTokenSet) {
myLexer = lexer;
myIdentifierTokenSet = identifierTokenSet;
myCommentTokenSet = commentTokenSet;
myLiteralTokenSet = literalTokenSet;
mySkipCodeContextTokenSet = skipCodeContextTokenSet;
}
public void processWords(CharSequence fileText, Processor<WordOccurrence> processor) {
myLexer.start(fileText);
WordOccurrence occurrence = null; // shared occurrence
while (myLexer.getTokenType() != null) {
final IElementType type = myLexer.getTokenType();
IElementType type;
while ((type = myLexer.getTokenType()) != null) {
if (myIdentifierTokenSet.contains(type)) {
if (occurrence == null) {
occurrence = new WordOccurrence(fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.CODE);
@@ -70,9 +87,7 @@ public class DefaultWordsScanner implements WordsScanner {
else if (myLiteralTokenSet.contains(type)) {
if (!stripWords(processor, fileText, myLexer.getTokenStart(),myLexer.getTokenEnd(),WordOccurrence.Kind.LITERALS,occurrence, myMayHaveFileRefsInLiterals)) return;
}
else {
// process all word-like characters as words
// Plugin writers may have (Maximka in JavaScript especially) some keyword token types omitted from the identifierTokenSet
else if (!mySkipCodeContextTokenSet.contains(type)) {
if (!stripWords(processor, fileText, myLexer.getTokenStart(), myLexer.getTokenEnd(), WordOccurrence.Kind.CODE, occurrence, false)) return;
}
myLexer.advance();
plugins/groovy/src/org/jetbrains/plugins/groovy/highlighter/GroovyTodoIndexer.java
+1 -4
View File
@@ -15,13 +15,11 @@
*/
package org.jetbrains.plugins.groovy.highlighter;
import com.intellij.lexer.FilterLexer;
import com.intellij.lexer.Lexer;
import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
import com.intellij.psi.impl.cache.impl.todo.LexerBasedTodoIndexer;
import org.jetbrains.plugins.groovy.lang.lexer.GroovyFilterLexer;
import org.jetbrains.plugins.groovy.lang.lexer.GroovyLexer;
import org.jetbrains.plugins.groovy.lang.lexer.TokenSets;
/**
* @author Maxim.Medvedev
@@ -29,7 +27,6 @@ import org.jetbrains.plugins.groovy.lang.lexer.TokenSets;
public class GroovyTodoIndexer extends LexerBasedTodoIndexer {
@Override
public Lexer createLexer(OccurrenceConsumer consumer) {
final GroovyFilterLexer groovyFilterLexer = new GroovyFilterLexer(new GroovyLexer(), consumer);
return new FilterLexer(groovyFilterLexer, new FilterLexer.SetFilter(TokenSets.WHITE_SPACES_OR_COMMENTS));
return new GroovyFilterLexer(new GroovyLexer(), consumer);
}
}
plugins/properties/src/com/intellij/psi/impl/cache/impl/idCache/PropertiesIdIndexer.java
Vendored
+1 -8
View File
@@ -16,26 +16,19 @@
package com.intellij.psi.impl.cache.impl.idCache;
import com.intellij.lang.properties.parsing.PropertiesLexer;
import com.intellij.lexer.FilterLexer;
import com.intellij.lexer.Lexer;
import com.intellij.psi.TokenType;
import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer;
import com.intellij.psi.tree.TokenSet;
/**
* @author Maxim.Mossienko
*/
public class PropertiesIdIndexer extends LexerBasedIdIndexer {
private static TokenSet WHITE_SPACE_SET = TokenSet.create(TokenType.WHITE_SPACE);
public Lexer createLexer(final OccurrenceConsumer consumer) {
return createIndexingLexer(consumer);
}
static Lexer createIndexingLexer(OccurrenceConsumer consumer) {
final PropertiesFilterLexer propsLexer =
new PropertiesFilterLexer(new PropertiesLexer(), consumer);
return new FilterLexer(propsLexer, new FilterLexer.SetFilter(WHITE_SPACE_SET));
return new PropertiesFilterLexer(new PropertiesLexer(), consumer);
}
}
xml/impl/src/com/intellij/psi/impl/cache/impl/idCache/XHtmlFilterLexer.java
Vendored
+4 -3
View File
@@ -32,7 +32,7 @@ public class XHtmlFilterLexer extends BaseFilterLexer {
}
public void advance() {
final IElementType tokenType = getDelegate().getTokenType();
final IElementType tokenType = myDelegate.getTokenType();
if (tokenType == XmlElementType.XML_COMMENT_CHARACTERS) {
scanWordsInToken(UsageSearchContext.IN_COMMENTS, false, false);
@@ -51,11 +51,12 @@ public class XHtmlFilterLexer extends BaseFilterLexer {
false);
if (inComments) advanceTodoItemCountsInToken();
} else {
}
else if (!XmlFilterLexer.ourNoWordsTokenSet.contains(tokenType)) {
scanWordsInToken(UsageSearchContext.IN_PLAIN_TEXT, false, false);
}
getDelegate().advance();
myDelegate.advance();
}
}
xml/impl/src/com/intellij/psi/impl/cache/impl/idCache/XmlFilterLexer.java
Vendored
+24 -3
View File
@@ -16,20 +16,41 @@
package com.intellij.psi.impl.cache.impl.idCache;
import com.intellij.lexer.Lexer;
import com.intellij.psi.TokenType;
import com.intellij.psi.impl.cache.impl.BaseFilterLexer;
import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
import com.intellij.psi.search.UsageSearchContext;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.psi.xml.XmlElementType;
import com.intellij.psi.xml.XmlTokenType;
public class XmlFilterLexer extends BaseFilterLexer {
static final TokenSet ourNoWordsTokenSet = TokenSet.create(
XmlTokenType.TAG_WHITE_SPACE,
TokenType.WHITE_SPACE,
XmlTokenType.XML_REAL_WHITE_SPACE,
XmlTokenType.XML_EQ,
XmlTokenType.XML_ATTRIBUTE_VALUE_START_DELIMITER,
XmlTokenType.XML_ATTRIBUTE_VALUE_END_DELIMITER,
XmlTokenType.XML_START_TAG_START,
XmlTokenType.XML_EMPTY_ELEMENT_END,
XmlTokenType.XML_END_TAG_START,
XmlTokenType.XML_TAG_END,
XmlTokenType.XML_DOCTYPE_END,
XmlTokenType.XML_COMMENT_START,
XmlTokenType.XML_COMMENT_END,
XmlTokenType.XML_PI_START,
XmlTokenType.XML_PI_END,
XmlTokenType.XML_CDATA_END
);
public XmlFilterLexer(Lexer originalLexer, OccurrenceConsumer table) {
super(originalLexer, table);
}
public void advance() {
final IElementType tokenType = getDelegate().getTokenType();
final IElementType tokenType = myDelegate.getTokenType();
if (tokenType == XmlElementType.XML_COMMENT_CHARACTERS) {
scanWordsInToken(UsageSearchContext.IN_COMMENTS, false, false);
@@ -48,10 +69,10 @@ public class XmlFilterLexer extends BaseFilterLexer {
else if (tokenType == XmlElementType.XML_TEXT) {
scanWordsInToken(UsageSearchContext.IN_PLAIN_TEXT | UsageSearchContext.IN_FOREIGN_LANGUAGES, false, false);
}
else {
else if (!ourNoWordsTokenSet.contains(tokenType)) {
scanWordsInToken(UsageSearchContext.IN_PLAIN_TEXT, false, false);
}
getDelegate().advance();
myDelegate.advance();
}
}