From e5d9dff7c04be337e14d1d4b377507a83e610b5a Mon Sep 17 00:00:00 2001 From: peter Date: Mon, 10 Mar 2014 18:38:42 +0100 Subject: [PATCH] add -> keyword to cpp and make sure it's lexed correctly (IDEA-121906) --- .../src/CommunityFileTypes.xml | 1 + .../completion/HippieCompletionTest.groovy | 9 +++++++ .../custom/CustomFileTypeLexer.java | 2 +- .../custom/tokens/IdentifierParser.java | 13 ++++++---- .../custom/tokens/KeywordParser.java | 25 +++++++++++-------- .../src/FileTypeManager.xml | 1 + .../custom/CustomFileTypeLexerTest.groovy | 10 ++++++++ 7 files changed, 45 insertions(+), 16 deletions(-) diff --git a/community-resources/src/CommunityFileTypes.xml b/community-resources/src/CommunityFileTypes.xml index e15ce6cff542..b46643afa562 100644 --- a/community-resources/src/CommunityFileTypes.xml +++ b/community-resources/src/CommunityFileTypes.xml @@ -60,6 +60,7 @@ + diff --git a/java/java-tests/testSrc/com/intellij/codeInsight/completion/HippieCompletionTest.groovy b/java/java-tests/testSrc/com/intellij/codeInsight/completion/HippieCompletionTest.groovy index 4cad78b0c467..7de79b533944 100644 --- a/java/java-tests/testSrc/com/intellij/codeInsight/completion/HippieCompletionTest.groovy +++ b/java/java-tests/testSrc/com/intellij/codeInsight/completion/HippieCompletionTest.groovy @@ -189,7 +189,16 @@ class Xoo { class Xoo { } ''' + } + public void "test cpp indirection"() { + myFixture.configureByText "a.c", '''f +foo->bar +''' + complete() + myFixture.checkResult '''foo +foo->bar +''' } private void complete() { diff --git a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/CustomFileTypeLexer.java b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/CustomFileTypeLexer.java index da784dfbad22..d253ace0930c 100644 --- a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/CustomFileTypeLexer.java +++ b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/CustomFileTypeLexer.java @@ -50,7 +50,7 @@ public final class CustomFileTypeLexer extends AbstractCustomLexer { } }; - final IdentifierParser identifierParser = new IdentifierParser(); + final IdentifierParser identifierParser = new IdentifierParser(parser); final QuotedStringParser quotedStringParser = new QuotedStringParser("\"", CustomHighlighterTokenType.STRING, table.isHasStringEscapes()); diff --git a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java index 57b29ae806a8..8fc92e722e0f 100644 --- a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java +++ b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/IdentifierParser.java @@ -23,7 +23,10 @@ import com.intellij.psi.tree.IElementType; * @author dsl */ public class IdentifierParser extends TokenParser { - public IdentifierParser() { + private final KeywordParser myKeywordParser; + + public IdentifierParser(KeywordParser keywordParser) { + myKeywordParser = keywordParser; } @Override @@ -31,15 +34,15 @@ public class IdentifierParser extends TokenParser { if (!Character.isJavaIdentifierStart(myBuffer.charAt(position))) return false; final int start = position; for (position++; position < myEndOffset; position++) { - final char c = myBuffer.charAt(position); - if (!isIdentifierPart(c)) break; + if (!isIdentifierPart(position)) break; } IElementType tokenType = CustomHighlighterTokenType.IDENTIFIER; myTokenInfo.updateData(start, position, tokenType); return true; } - protected boolean isIdentifierPart(final char c) { - return Character.isJavaIdentifierPart(c) || c == '-'; + private boolean isIdentifierPart(int position) { + if (myBuffer.charAt(position) == '-') return !myKeywordParser.hasToken(position, myBuffer, null); + return KeywordParser.isWordPart(position, myBuffer); } } diff --git a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java index f45032c23f05..b210305c2489 100644 --- a/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java +++ b/platform/indexing-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java @@ -23,6 +23,7 @@ import com.intellij.psi.tree.IElementType; import com.intellij.util.containers.CharTrie; import gnu.trove.THashSet; import gnu.trove.TIntHashSet; +import org.jetbrains.annotations.Nullable; import java.util.ArrayList; import java.util.List; @@ -63,10 +64,9 @@ public class KeywordParser { return result; } - public boolean hasToken(int position, CharSequence myBuffer, TokenInfo myTokenInfo) { + public boolean hasToken(int position, CharSequence myBuffer, @Nullable TokenInfo tokenInfo) { int index = 0; int offset = position; - boolean found = false; while (offset < myBuffer.length()) { char c = myBuffer.charAt(offset++); int nextIndex = myTrie.findSubNode(index, myIgnoreCase ? Character.toUpperCase(c) : c); @@ -79,24 +79,29 @@ public class KeywordParser { String testKeyword = myIgnoreCase ? StringUtil.toUpperCase(keyword) : keyword; for (int i = 0; i < CustomHighlighterTokenType.KEYWORD_TYPE_COUNT; i++) { if (myKeywordSets.get(i).contains(testKeyword)) { - myTokenInfo.updateData(position, position + keyword.length(), getToken(i)); - found = true; - break; + if (tokenInfo != null) { + tokenInfo.updateData(position, position + keyword.length(), getToken(i)); + } + return true; } } } } - return found; + return false; } - private static boolean isWordEnd(int offset, CharSequence myBuffer) { - if (offset == myBuffer.length()) { + private static boolean isWordEnd(int offset, CharSequence sequence) { + if (offset == sequence.length()) { return true; } + + return !isWordPart(offset - 1, sequence) || !isWordPart(offset, sequence); + } - char ch = myBuffer.charAt(offset); - return ch != '-' && ch != '_' && !Character.isLetterOrDigit(ch); + static boolean isWordPart(int offset, CharSequence sequence) { + char ch = sequence.charAt(offset); + return ch == '-' || Character.isJavaIdentifierPart(ch); } private static IElementType getToken(int keywordSetIndex) { diff --git a/platform/platform-resources/src/FileTypeManager.xml b/platform/platform-resources/src/FileTypeManager.xml index a381b19a9a5f..2a643ddc1661 100644 --- a/platform/platform-resources/src/FileTypeManager.xml +++ b/platform/platform-resources/src/FileTypeManager.xml @@ -60,6 +60,7 @@ + diff --git a/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy b/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy index 0ac9a1e03cbf..7f13b49765c9 100644 --- a/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy +++ b/platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy @@ -283,6 +283,16 @@ IDENTIFIER ('k') ''' } + public void testCpp() { + SyntaxTable table = new SyntaxTable() + table.addKeyword1('->') + doTest table, "foo->bar", '''\ +IDENTIFIER ('foo') +KEYWORD_1 ('->') +IDENTIFIER ('bar') +''' + } + public void testNumber() { doTest createPropTable(), "1.23=1.24", '''\ NUMBER ('1.23')