MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

java: lexers united; lexer creation extracted

Browse Source
This commit is contained in:
Roman Shevchenko
2013-06-07 12:00:15 +04:00
parent 03e5528a24
commit a0f88a0987
31 changed files with 1606 additions and 840 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/idea-ui/src/com/intellij/ide/util/importProject/JavaModuleInsight.java
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2011 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ package com.intellij.ide.util.importProject;
import com.intellij.ide.util.projectWizard.importSources.DetectedProjectRoot;
import com.intellij.ide.util.projectWizard.importSources.JavaModuleSourceRoot;
import com.intellij.ide.util.projectWizard.importSources.JavaSourceRootDetectionUtil;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.module.StdModuleTypes;
import com.intellij.openapi.progress.ProgressIndicator;
@@ -38,13 +38,13 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
public class JavaModuleInsight extends ModuleInsight {
private final JavaLexer myLexer;
private final Lexer myLexer;
public JavaModuleInsight(@Nullable final ProgressIndicator progress,
Set<String> existingModuleNames,
Set<String> existingProjectLibraryNames) {
super(progress, existingModuleNames, existingProjectLibraryNames);
myLexer = new JavaLexer(LanguageLevel.JDK_1_5);
myLexer = JavaParserDefinition.createLexer(LanguageLevel.JDK_1_5);
}
@Override
java/idea-ui/src/com/intellij/ide/util/projectWizard/importSources/JavaSourceRootDetectionUtil.java
+3 -11
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,32 +16,25 @@
package com.intellij.ide.util.projectWizard.importSources;
import com.intellij.ide.util.importProject.RootDetectionProcessor;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.impl.source.tree.ElementType;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.NullableFunction;
import com.intellij.util.StringBuilderSpinAllocator;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.text.CharArrayCharSequence;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class JavaSourceRootDetectionUtil {
private static final TokenSet JAVA_FILE_FIRST_TOKEN_SET = TokenSet.orSet(
ElementType.MODIFIER_BIT_SET,
@@ -76,10 +69,9 @@ public class JavaSourceRootDetectionUtil {
return result.values();
}
@Nullable
public static String getPackageName(CharSequence text) {
Lexer lexer = new JavaLexer(LanguageLevel.JDK_1_3);
Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.JDK_1_3);
lexer.start(text);
skipWhiteSpaceAndComments(lexer);
final IElementType firstToken = lexer.getTokenType();
java/java-impl/src/com/intellij/lexer/EscapedJavaLexer.java
-165
View File
@@ -1,165 +0,0 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.tree.IElementType;
/**
* Used to process scriptlet code in JSP attribute values like this:
* attribute="<%=texts.get(\"Blabla\")%>"
*/
public class EscapedJavaLexer extends LexerBase {
private char mySurroundingQuote;
private final JavaLexer myJavaLexer;
private CharSequence myBuffer;
private int myBufferEnd;
private int myCurOffset;
private IElementType myTokenType = null;
private int myTokenEnd;
public EscapedJavaLexer(char surroundingQuote, LanguageLevel languageLevel) {
mySurroundingQuote = surroundingQuote;
myJavaLexer = new JavaLexer(languageLevel);
}
public char getSurroundingQuote() {
return mySurroundingQuote;
}
public void setSurroundingQuote(char surroundingQuote) {
mySurroundingQuote = surroundingQuote;
}
public void start(CharSequence buffer, int startOffset, int endOffset, int state) {
myBuffer = buffer;
myCurOffset = startOffset;
myTokenEnd = startOffset;
myBufferEnd = endOffset;
myTokenType = null;
}
public CharSequence getBufferSequence() {
return myBuffer;
}
public int getState() {
return 0;
}
public IElementType getTokenType() {
locateToken();
return myTokenType;
}
public final int getTokenStart(){
locateToken();
return myCurOffset;
}
public final int getTokenEnd(){
locateToken();
return myTokenEnd;
}
public final void advance(){
locateToken();
myTokenType = null;
myCurOffset = myTokenEnd;
}
public final int getBufferEnd(){
return myBufferEnd;
}
private void locateToken() {
if (myTokenType != null) return;
if (myCurOffset >= myBufferEnd) return;
boolean esc = false;
int offset = myCurOffset;
int state = 0; // 0 -- start/end
// 1 -- inside string
// 2 -- after escape (/) in string literal
char literalStarter = 0;
do {
if (offset >= myBufferEnd) break;
char c = myBuffer.charAt(offset);
boolean wasEsc = esc;
esc = false;
if (c == '\\') {
if (!wasEsc) {
esc = true;
}
else {
state = 2;
}
}
else if (state == 0) {
if (c == '\'' || c == '\"') {
literalStarter = c;
state = 1;
}
}
else if (state == 1) {
if (c == literalStarter) {
state = 0;
offset++;
break;
}
}
else if (state == 2) {
state = 1;
}
if (!esc && state == 0) {
break;
}
offset++;
}
while (true);
if(offset >= myBufferEnd - 1) state = 0;
switch (state){
case 0:
if(offset == myCurOffset){
myJavaLexer.start(myBuffer, myCurOffset, myBufferEnd);
myTokenType = myJavaLexer.getTokenType();
myTokenEnd = myJavaLexer.getTokenEnd();
}
else {
myTokenType = literalStarter == '\"' ? JavaTokenType.STRING_LITERAL : JavaTokenType.CHARACTER_LITERAL;
myTokenEnd = offset;
}
break;
case 1:
myTokenType = literalStarter == '\"' ? JavaTokenType.STRING_LITERAL : JavaTokenType.CHARACTER_LITERAL;
myTokenEnd = offset;
break;
default:
myTokenType = JavaTokenType.BAD_CHARACTER;
myTokenEnd = offset;
break;
}
}
}
java/java-impl/src/com/intellij/lexer/JavaHighlightingLexer.java
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,26 +15,26 @@
*/
package com.intellij.lexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaDocTokenType;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.impl.source.tree.JavaDocElementType;
import com.intellij.psi.tree.IElementType;
/**
* @author max
*/
public class JavaHighlightingLexer extends LayeredLexer {
public JavaHighlightingLexer(LanguageLevel languageLevel) {
super(new JavaLexer(languageLevel));
super(JavaParserDefinition.createLexer(languageLevel));
registerSelfStoppingLayer(new StringLiteralLexer('\"', JavaTokenType.STRING_LITERAL),
new IElementType[]{JavaTokenType.STRING_LITERAL}, IElementType.EMPTY_ARRAY);
registerSelfStoppingLayer(new StringLiteralLexer('\'', JavaTokenType.STRING_LITERAL),
new IElementType[]{JavaTokenType.CHARACTER_LITERAL}, IElementType.EMPTY_ARRAY);
LayeredLexer docLexer = new LayeredLexer(new JavaDocLexer(languageLevel.isAtLeast(LanguageLevel.JDK_1_5)));
LayeredLexer docLexer = new LayeredLexer(JavaParserDefinition.createDocLexer(languageLevel));
HtmlHighlightingLexer lexer = new HtmlHighlightingLexer();
lexer.setHasNoEmbeddments(true);
java/java-impl/src/com/intellij/psi/formatter/java/JavaSpacePropertyProcessor.java
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ package com.intellij.psi.formatter.java;
import com.intellij.formatting.Spacing;
import com.intellij.lang.ASTNode;
import com.intellij.lang.java.JavaLanguage;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
@@ -1613,7 +1613,7 @@ public class JavaSpacePropertyProcessor extends JavaElementVisitor {
if (res == null) {
if (!checkToken(token1) || !checkToken(token2)) return true;
String text = token1.getText() + token2.getText();
Lexer lexer = new JavaLexer(LanguageLevel.HIGHEST);
Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
lexer.start(text);
boolean canMerge = lexer.getTokenType() == type1;
lexer.advance();
@@ -1625,7 +1625,7 @@ public class JavaSpacePropertyProcessor extends JavaElementVisitor {
}
private static boolean checkToken(final ASTNode token1) {
Lexer lexer = new JavaLexer(LanguageLevel.HIGHEST);
Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
final String text = token1.getText();
lexer.start(text);
if (lexer.getTokenType() != token1.getElementType()) return false;
java/java-impl/src/com/intellij/psi/impl/cache/impl/idCache/JavaIdIndexer.java
Vendored
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
*/
package com.intellij.psi.impl.cache.impl.idCache;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.impl.cache.impl.OccurrenceConsumer;
@@ -28,7 +28,7 @@ public class JavaIdIndexer extends LexerBasedIdIndexer {
}
public static Lexer createIndexingLexer(OccurrenceConsumer consumer) {
final JavaLexer javaLexer = new JavaLexer(LanguageLevel.JDK_1_3);
Lexer javaLexer = JavaParserDefinition.createLexer(LanguageLevel.JDK_1_3);
return new JavaFilterLexer(javaLexer, consumer);
}
}
java/java-impl/src/com/intellij/psi/impl/search/JavaIndexPatternBuilder.java
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
*/
package com.intellij.psi.impl.search;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.psi.JavaDocTokenType;
import com.intellij.psi.JavaTokenType;
@@ -40,7 +40,7 @@ public class JavaIndexPatternBuilder implements IndexPatternBuilder {
@Nullable
public Lexer getIndexingLexer(final PsiFile file) {
if (file instanceof PsiJavaFile && !(file instanceof JspFile)) {
return new JavaLexer(((PsiJavaFile)file).getLanguageLevel());
return JavaParserDefinition.createLexer(((PsiJavaFile)file).getLanguageLevel());
}
return null;
}
java/java-psi-api/src/com/intellij/lexer/JavaDocLexer.java
+3 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,9 +15,8 @@
*/
package com.intellij.lexer;
/**
* @author yole
*/
/** @deprecated use com.intellij.lang.java.JavaDocLexer (to remove in IDEA 14) */
@SuppressWarnings("UnusedDeclaration")
public class JavaDocLexer extends DocCommentLexer {
public JavaDocLexer(final boolean isJdk15Enabled) {
super(JavaDocTokenTypes.INSTANCE, isJdk15Enabled);
java/java-psi-impl/src/com/intellij/lang/java/JavaParserDefinition.java
+28 -18
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2011 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,7 +19,8 @@ import com.intellij.lang.ASTNode;
import com.intellij.lang.LanguageUtil;
import com.intellij.lang.ParserDefinition;
import com.intellij.lang.PsiParser;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.lexer.JavaDocLexer;
import com.intellij.lang.java.lexer.JavaLexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.roots.LanguageLevelProjectExtension;
@@ -34,6 +35,7 @@ import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.IFileElementType;
import com.intellij.psi.tree.TokenSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* @author max
@@ -41,14 +43,19 @@ import org.jetbrains.annotations.NotNull;
public class JavaParserDefinition implements ParserDefinition {
@Override
@NotNull
public Lexer createLexer(final Project project) {
final LanguageLevel languageLevel = LanguageLevelProjectExtension.getInstance(project).getLanguageLevel();
return createLexer(languageLevel);
public Lexer createLexer(@Nullable Project project) {
LanguageLevel level = project != null ? LanguageLevelProjectExtension.getInstance(project).getLanguageLevel() : LanguageLevel.HIGHEST;
return createLexer(level);
}
@NotNull
public static Lexer createLexer(final LanguageLevel languageLevel) {
return new JavaLexer(languageLevel);
public static Lexer createLexer(@NotNull LanguageLevel level) {
return new JavaLexer(level);
}
@NotNull
public static Lexer createDocLexer(@NotNull LanguageLevel level) {
return new JavaDocLexer(level);
}
@Override
@@ -98,24 +105,27 @@ public class JavaParserDefinition implements ParserDefinition {
@Override
public SpaceRequirements spaceExistanceTypeBetweenTokens(ASTNode left, ASTNode right) {
final PsiFile containingFile = left.getTreeParent().getPsi().getContainingFile();
final Lexer lexer;
if(containingFile instanceof PsiJavaFile)
lexer = new JavaLexer(((PsiJavaFile)containingFile).getLanguageLevel());
else lexer = new JavaLexer(LanguageLevel.HIGHEST);
if(right.getElementType() == JavaDocTokenType.DOC_TAG_VALUE_SHARP_TOKEN) return SpaceRequirements.MUST_NOT;
if(left.getElementType() == JavaDocTokenType.DOC_TAG_VALUE_SHARP_TOKEN) return SpaceRequirements.MUST_NOT;
final SpaceRequirements spaceRequirements = LanguageUtil.canStickTokensTogetherByLexer(left, right, lexer);
if(left.getElementType() == JavaTokenType.END_OF_LINE_COMMENT) return SpaceRequirements.MUST_LINE_BREAK;
if (right.getElementType() == JavaDocTokenType.DOC_TAG_VALUE_SHARP_TOKEN ||
left.getElementType() == JavaDocTokenType.DOC_TAG_VALUE_SHARP_TOKEN) {
return SpaceRequirements.MUST_NOT;
}
if(left.getElementType() == JavaDocTokenType.DOC_COMMENT_DATA) {
PsiFile containingFile = left.getTreeParent().getPsi().getContainingFile();
LanguageLevel level = containingFile instanceof PsiJavaFile? ((PsiJavaFile)containingFile).getLanguageLevel() : LanguageLevel.HIGHEST;
Lexer lexer = createLexer(level);
SpaceRequirements spaceRequirements = LanguageUtil.canStickTokensTogetherByLexer(left, right, lexer);
if (left.getElementType() == JavaTokenType.END_OF_LINE_COMMENT) {
return SpaceRequirements.MUST_LINE_BREAK;
}
if (left.getElementType() == JavaDocTokenType.DOC_COMMENT_DATA) {
String text = left.getText();
if (text.length() > 0 && Character.isWhitespace(text.charAt(text.length() - 1))) {
return SpaceRequirements.MAY;
}
}
if(right.getElementType() == JavaDocTokenType.DOC_COMMENT_DATA) {
if (right.getElementType() == JavaDocTokenType.DOC_COMMENT_DATA) {
String text = right.getText();
if (text.length() > 0 && Character.isWhitespace(text.charAt(0))) {
return SpaceRequirements.MAY;
java/java-psi-impl/src/com/intellij/lang/java/lexer/JavaDocLexer.java
+179
View File
@@ -0,0 +1,179 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lang.java.lexer;
import com.intellij.lexer.DocCommentTokenTypes;
import com.intellij.lexer.JavaDocTokenTypes;
import com.intellij.lexer.LexerBase;
import com.intellij.lexer.MergingLexerAdapter;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.text.CharArrayUtil;
import org.jetbrains.annotations.NotNull;
import java.io.IOException;
public class JavaDocLexer extends MergingLexerAdapter {
public JavaDocLexer(@NotNull LanguageLevel level) {
this(JavaDocTokenTypes.INSTANCE, level.isAtLeast(LanguageLevel.JDK_1_5));
}
private JavaDocLexer(DocCommentTokenTypes tokenTypes, boolean isJdk15Enabled) {
super(new AsteriskStripperLexer(new _JavaDocLexer(isJdk15Enabled, tokenTypes), tokenTypes),
tokenTypes.spaceCommentsTokenSet());
}
private static class AsteriskStripperLexer extends LexerBase {
private final _JavaDocLexer myFlex;
private final DocCommentTokenTypes myTokenTypes;
private CharSequence myBuffer;
private int myBufferIndex;
private int myBufferEndOffset;
private int myTokenEndOffset;
private int myState;
private IElementType myTokenType;
private boolean myAfterLineBreak;
private boolean myInLeadingSpace;
public AsteriskStripperLexer(final _JavaDocLexer flex, final DocCommentTokenTypes tokenTypes) {
myFlex = flex;
myTokenTypes = tokenTypes;
}
@Override
public final void start(CharSequence buffer, int startOffset, int endOffset, int initialState) {
myBuffer = buffer;
myBufferIndex = startOffset;
myBufferEndOffset = endOffset;
myTokenType = null;
myTokenEndOffset = startOffset;
myFlex.reset(myBuffer, startOffset, endOffset, initialState);
}
@Override
public int getState() {
return myState;
}
@Override
public CharSequence getBufferSequence() {
return myBuffer;
}
@Override
public int getBufferEnd() {
return myBufferEndOffset;
}
@Override
public final IElementType getTokenType() {
locateToken();
return myTokenType;
}
@Override
public final int getTokenStart() {
locateToken();
return myBufferIndex;
}
@Override
public final int getTokenEnd() {
locateToken();
return myTokenEndOffset;
}
@Override
public final void advance() {
locateToken();
myTokenType = null;
}
protected final void locateToken() {
if (myTokenType != null) return;
_locateToken();
if (myTokenType == myTokenTypes.space()) {
myAfterLineBreak = CharArrayUtil.containLineBreaks(myBuffer, getTokenStart(), getTokenEnd());
}
}
private void _locateToken() {
if (myTokenEndOffset == myBufferEndOffset) {
myTokenType = null;
myBufferIndex = myBufferEndOffset;
return;
}
myBufferIndex = myTokenEndOffset;
if (myAfterLineBreak) {
myAfterLineBreak = false;
while (myTokenEndOffset < myBufferEndOffset && myBuffer.charAt(myTokenEndOffset) == '*' &&
(myTokenEndOffset + 1 >= myBufferEndOffset || myBuffer.charAt(myTokenEndOffset + 1) != '/')) {
myTokenEndOffset++;
}
myInLeadingSpace = true;
if (myBufferIndex < myTokenEndOffset) {
myTokenType = myTokenTypes.commentLeadingAsterisks();
return;
}
}
if (myInLeadingSpace) {
myInLeadingSpace = false;
boolean lf = false;
while (myTokenEndOffset < myBufferEndOffset && Character.isWhitespace(myBuffer.charAt(myTokenEndOffset))) {
if (myBuffer.charAt(myTokenEndOffset) == '\n') lf = true;
myTokenEndOffset++;
}
final int state = myFlex.yystate();
if (state == _JavaDocLexer.COMMENT_DATA ||
myTokenEndOffset < myBufferEndOffset && (myBuffer.charAt(myTokenEndOffset) == '@' ||
myBuffer.charAt(myTokenEndOffset) == '{' ||
myBuffer.charAt(myTokenEndOffset) == '\"' ||
myBuffer.charAt(myTokenEndOffset) == '<')) {
myFlex.yybegin(_JavaDocLexer.COMMENT_DATA_START);
}
if (myBufferIndex < myTokenEndOffset) {
myTokenType = lf || state == _JavaDocLexer.PARAM_TAG_SPACE || state == _JavaDocLexer.TAG_DOC_SPACE || state == _JavaDocLexer.INLINE_TAG_NAME || state == _JavaDocLexer.DOC_TAG_VALUE_IN_PAREN
? myTokenTypes.space()
: myTokenTypes.commentData();
return;
}
}
flexLocateToken();
}
private void flexLocateToken() {
try {
myState = myFlex.yystate();
myFlex.goTo(myBufferIndex);
myTokenType = myFlex.advance();
myTokenEndOffset = myFlex.getTokenEnd();
}
catch (IOException e) {
// Can't be
}
}
}
}
java/java-psi-impl/src/com/intellij/lang/java/lexer/JavaLexer.java
+429
View File
@@ -0,0 +1,429 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lang.java.lexer;
import com.intellij.lexer.LexerBase;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.TokenType;
import com.intellij.psi.impl.source.tree.JavaDocElementType;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.text.CharArrayUtil;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
import java.io.IOException;
import java.util.Set;
public class JavaLexer extends LexerBase {
private static final HashTable[] TABLES = new HashTable[]{
new HashTable(LanguageLevel.JDK_1_5),
new HashTable(LanguageLevel.JDK_1_4),
new HashTable(LanguageLevel.JDK_1_3)
};
private static HashTable getTable(final LanguageLevel level) {
for (HashTable table : TABLES) {
if (level.isAtLeast(table.myLevel)) {
return table;
}
}
throw new IllegalArgumentException("Unsupported level: " + level);
}
public static boolean isKeyword(String id, LanguageLevel level) {
return getTable(level).contains(id);
}
private final _JavaLexer myFlexLexer;
private final HashTable myTable;
private CharSequence myBuffer;
private char[] myBufferArray;
private int myBufferIndex;
private int myBufferEndOffset;
private int myTokenEndOffset; // positioned after the last symbol of the current token
private IElementType myTokenType;
public JavaLexer(@NotNull final LanguageLevel level) {
myFlexLexer = new _JavaLexer(level);
myTable = getTable(level);
}
private static final class HashTable {
private static final int NUM_ENTRIES = 999;
private static final Logger LOG = Logger.getInstance("com.intellij.Lexer.JavaLexer");
private final LanguageLevel myLevel;
private final char[][] myTable = new char[NUM_ENTRIES][];
private final IElementType[] myKeywords = new IElementType[NUM_ENTRIES];
private final Set<String> myKeywordsInSet = new THashSet<String>();
private void add(String s, IElementType tokenType) {
char[] chars = s.toCharArray();
int hashCode = chars[0] * 2;
for (int j = 1; j < chars.length; j++) {
hashCode += chars[j];
}
int modHashCode = hashCode % NUM_ENTRIES;
LOG.assertTrue(myTable[modHashCode] == null);
myTable[modHashCode] = chars;
myKeywords[modHashCode] = tokenType;
myKeywordsInSet.add(s);
}
public boolean contains(String s) {
return myKeywordsInSet.contains(s);
}
private boolean contains(int hashCode, final char[] bufferArray, final CharSequence buffer, int offset) {
int modHashCode = hashCode % NUM_ENTRIES;
final char[] kwd = myTable[modHashCode];
if (kwd == null) return false;
if (bufferArray != null) {
for (int j = 0; j < kwd.length; j++) {
if (bufferArray[j + offset] != kwd[j]) return false;
}
} else {
for (int j = 0; j < kwd.length; j++) {
if (buffer.charAt(j + offset) != kwd[j]) return false;
}
}
return true;
}
private IElementType getTokenType(int hashCode) {
return myKeywords[hashCode % NUM_ENTRIES];
}
@SuppressWarnings({"HardCodedStringLiteral"})
private HashTable(final LanguageLevel level) {
myLevel = level;
if (level.isAtLeast(LanguageLevel.JDK_1_4)) {
add("assert", JavaTokenType.ASSERT_KEYWORD);
if (level.isAtLeast(LanguageLevel.JDK_1_5)) {
add("enum", JavaTokenType.ENUM_KEYWORD);
}
}
add("abstract", JavaTokenType.ABSTRACT_KEYWORD);
add("default", JavaTokenType.DEFAULT_KEYWORD);
add("if", JavaTokenType.IF_KEYWORD);
add("private", JavaTokenType.PRIVATE_KEYWORD);
add("this", JavaTokenType.THIS_KEYWORD);
add("boolean", JavaTokenType.BOOLEAN_KEYWORD);
add("do", JavaTokenType.DO_KEYWORD);
add("implements", JavaTokenType.IMPLEMENTS_KEYWORD);
add("protected", JavaTokenType.PROTECTED_KEYWORD);
add("throw", JavaTokenType.THROW_KEYWORD);
add("break", JavaTokenType.BREAK_KEYWORD);
add("double", JavaTokenType.DOUBLE_KEYWORD);
add("import", JavaTokenType.IMPORT_KEYWORD);
add("public", JavaTokenType.PUBLIC_KEYWORD);
add("throws", JavaTokenType.THROWS_KEYWORD);
add("byte", JavaTokenType.BYTE_KEYWORD);
add("else", JavaTokenType.ELSE_KEYWORD);
add("instanceof", JavaTokenType.INSTANCEOF_KEYWORD);
add("return", JavaTokenType.RETURN_KEYWORD);
add("transient", JavaTokenType.TRANSIENT_KEYWORD);
add("case", JavaTokenType.CASE_KEYWORD);
add("extends", JavaTokenType.EXTENDS_KEYWORD);
add("int", JavaTokenType.INT_KEYWORD);
add("short", JavaTokenType.SHORT_KEYWORD);
add("try", JavaTokenType.TRY_KEYWORD);
add("catch", JavaTokenType.CATCH_KEYWORD);
add("final", JavaTokenType.FINAL_KEYWORD);
add("interface", JavaTokenType.INTERFACE_KEYWORD);
add("static", JavaTokenType.STATIC_KEYWORD);
add("void", JavaTokenType.VOID_KEYWORD);
add("char", JavaTokenType.CHAR_KEYWORD);
add("finally", JavaTokenType.FINALLY_KEYWORD);
add("long", JavaTokenType.LONG_KEYWORD);
add("strictfp", JavaTokenType.STRICTFP_KEYWORD);
add("volatile", JavaTokenType.VOLATILE_KEYWORD);
add("class", JavaTokenType.CLASS_KEYWORD);
add("float", JavaTokenType.FLOAT_KEYWORD);
add("native", JavaTokenType.NATIVE_KEYWORD);
add("super", JavaTokenType.SUPER_KEYWORD);
add("while", JavaTokenType.WHILE_KEYWORD);
add("const", JavaTokenType.CONST_KEYWORD);
add("for", JavaTokenType.FOR_KEYWORD);
add("new", JavaTokenType.NEW_KEYWORD);
add("switch", JavaTokenType.SWITCH_KEYWORD);
add("continue", JavaTokenType.CONTINUE_KEYWORD);
add("goto", JavaTokenType.GOTO_KEYWORD);
add("package", JavaTokenType.PACKAGE_KEYWORD);
add("synchronized", JavaTokenType.SYNCHRONIZED_KEYWORD);
add("true", JavaTokenType.TRUE_KEYWORD);
add("false", JavaTokenType.FALSE_KEYWORD);
add("null", JavaTokenType.NULL_KEYWORD);
}
}
@Override
public final void start(CharSequence buffer, int startOffset, int endOffset, int initialState) {
myBuffer = buffer;
myBufferArray = CharArrayUtil.fromSequenceWithoutCopying(buffer);
myBufferIndex = startOffset;
myBufferEndOffset = endOffset;
myTokenType = null;
myTokenEndOffset = startOffset;
myFlexLexer.reset(myBuffer, startOffset, endOffset, 0);
}
@Override
public int getState() {
return 0;
}
@Override
public final IElementType getTokenType() {
if (myTokenType == null) _locateToken();
return myTokenType;
}
@Override
public final int getTokenStart() {
return myBufferIndex;
}
@Override
public final int getTokenEnd() {
if (myTokenType == null) _locateToken();
return myTokenEndOffset;
}
@Override
public final void advance() {
if (myTokenType == null) _locateToken();
myTokenType = null;
}
private void _locateToken() {
if (myTokenEndOffset == myBufferEndOffset) {
myTokenType = null;
myBufferIndex = myBufferEndOffset;
return;
}
myBufferIndex = myTokenEndOffset;
final char c = myBufferArray != null ? myBufferArray[myBufferIndex]:myBuffer.charAt(myBufferIndex);
switch (c) {
default:
flexLocateToken();
break;
case ' ':
case '\t':
case '\n':
case '\r':
case '\f':
myTokenType = TokenType.WHITE_SPACE;
myTokenEndOffset = getWhitespaces(myBufferIndex + 1);
break;
case '/':
if (myBufferIndex + 1 >= myBufferEndOffset) {
myTokenType = JavaTokenType.DIV;
myTokenEndOffset = myBufferEndOffset;
}
else {
final char nextChar = myBufferArray != null ? myBufferArray[myBufferIndex + 1]:myBuffer.charAt(myBufferIndex + 1);
if (nextChar == '/') {
myTokenType = JavaTokenType.END_OF_LINE_COMMENT;
myTokenEndOffset = getLineTerminator(myBufferIndex + 2);
}
else if (nextChar == '*') {
if (myBufferIndex + 2 >= myBufferEndOffset ||
(myBufferArray != null ? myBufferArray[myBufferIndex + 2]:myBuffer.charAt(myBufferIndex + 2)) != '*' ||
(myBufferIndex + 3 < myBufferEndOffset &&
(myBufferArray != null ? myBufferArray[myBufferIndex + 3]:myBuffer.charAt(myBufferIndex + 3)) == '/')) {
myTokenType = JavaTokenType.C_STYLE_COMMENT;
myTokenEndOffset = getClosingComment(myBufferIndex + 2);
}
else {
myTokenType = JavaDocElementType.DOC_COMMENT;
myTokenEndOffset = getClosingComment(myBufferIndex + 3);
}
}
else if (c > 127 && Character.isJavaIdentifierStart(c)) {
myTokenEndOffset = getIdentifier(myBufferIndex + 1);
}
else {
flexLocateToken();
}
}
break;
case '"':
case '\'':
myTokenType = c == '"' ? JavaTokenType.STRING_LITERAL : JavaTokenType.CHARACTER_LITERAL;
myTokenEndOffset = getClosingParenthesis(myBufferIndex + 1, c);
}
if (myTokenEndOffset > myBufferEndOffset) {
myTokenEndOffset = myBufferEndOffset;
}
}
private int getWhitespaces(int pos) {
if (pos >= myBufferEndOffset) return myBufferEndOffset;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
while (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f') {
pos++;
if (pos == myBufferEndOffset) return pos;
c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
return pos;
}
private void flexLocateToken() {
try {
myFlexLexer.goTo(myBufferIndex);
myTokenType = myFlexLexer.advance();
myTokenEndOffset = myFlexLexer.getTokenEnd();
}
catch (IOException e) {
// Can't be
}
}
private int getClosingParenthesis(int offset, char c) {
int pos = offset;
final int lBufferEnd = myBufferEndOffset;
if (pos >= lBufferEnd) return lBufferEnd;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
char cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
while (true) {
while (cur != c && cur != '\n' && cur != '\r' && cur != '\\') {
pos++;
if (pos >= lBufferEnd) return lBufferEnd;
cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
if (cur == '\\') {
pos++;
if (pos >= lBufferEnd) return lBufferEnd;
cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
if (cur == '\n' || cur == '\r') continue;
pos++;
if (pos >= lBufferEnd) return lBufferEnd;
cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
else if (cur == c) {
break;
}
else {
pos--;
break;
}
}
return pos + 1;
}
private int getClosingComment(int offset) {
int pos = offset;
final int lBufferEnd = myBufferEndOffset;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
while (pos < lBufferEnd - 1) {
final char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
if (c == '*' && (lBufferArray != null ? lBufferArray[pos + 1]:lBuffer.charAt(pos + 1)) == '/') {
break;
}
pos++;
}
return pos + 2;
}
private int getLineTerminator(int offset) {
int pos = offset;
final int lBufferEnd = myBufferEndOffset;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
while (pos < lBufferEnd) {
final char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
if (c == '\r' || c == '\n') break;
pos++;
}
return pos;
}
private int getIdentifier(int offset) {
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
int hashCode = (lBufferArray != null ? lBufferArray[offset - 1]:lBuffer.charAt(offset - 1)) * 2;
final int lBufferEnd = myBufferEndOffset;
int pos = offset;
if (pos < lBufferEnd) {
char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
while (c >= 'a' && c <= 'z' ||
c >= 'A' && c <= 'Z' ||
c >= '0' && c <= '9' ||
c == '_' ||
c == '$' ||
c > 127 && Character.isJavaIdentifierPart(c)) {
pos++;
hashCode += c;
if (pos == lBufferEnd) break;
c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
}
if (myTable.contains(hashCode, lBufferArray, lBuffer, offset - 1)) {
myTokenType = myTable.getTokenType(hashCode);
}
else {
myTokenType = JavaTokenType.IDENTIFIER;
}
return pos;
}
@Override
public CharSequence getBufferSequence() {
return myBuffer;
}
@Override
public final int getBufferEnd() {
return myBufferEndOffset;
}
}
platform/lang-api/src/com/intellij/lexer/_JavaDocLexer.flex → java/java-psi-impl/src/com/intellij/lang/java/lexer/_JavaDocLexer.flex
+18 -9
View File
@@ -1,9 +1,11 @@
/* It's an automatically generated code. Do not modify it. */
package com.intellij.lexer;
package com.intellij.lang.java.lexer;
import com.intellij.lexer.DocCommentTokenTypes;
import com.intellij.lexer.FlexLexer;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.*;
@SuppressWarnings("ALL")
%%
%{
@@ -53,7 +55,6 @@ IDENTIFIER={ALPHA}({ALPHA}|{DIGIT}|[":.-"])*
%%
<YYINITIAL> "/**/" { return myTokenTypes.commentEnd(); }
<YYINITIAL> "/**" { yybegin(COMMENT_DATA_START); return myTokenTypes.commentStart(); }
<COMMENT_DATA_START> {WHITE_DOC_SPACE_CHAR}+ { return myTokenTypes.space(); }
<COMMENT_DATA> {WHITE_DOC_SPACE_NO_LR}+ { return myTokenTypes.commentData(); }
@@ -69,13 +70,22 @@ IDENTIFIER={ALPHA}({ALPHA}|{DIGIT}|[":.-"])*
<INLINE_TAG_NAME, COMMENT_DATA_START> "@param" { yybegin(PARAM_TAG_SPACE); return myTokenTypes.tagName(); }
<PARAM_TAG_SPACE> {WHITE_DOC_SPACE_CHAR}+ {yybegin(DOC_TAG_VALUE); return myTokenTypes.space();}
<DOC_TAG_VALUE> [\<] { if (myJdk15Enabled) {yybegin(DOC_TAG_VALUE_IN_LTGT); return myTokenTypes.tagValueLT();} else { yybegin(COMMENT_DATA); return myTokenTypes.commentData(); } }
<DOC_TAG_VALUE> [\<] {
if (myJdk15Enabled) {
yybegin(DOC_TAG_VALUE_IN_LTGT);
return myTokenTypes.tagValueLT();
}
else {
yybegin(COMMENT_DATA);
return myTokenTypes.commentData();
}
}
<DOC_TAG_VALUE_IN_LTGT> {IDENTIFIER} { return myTokenTypes.tagValueToken(); }
<DOC_TAG_VALUE_IN_LTGT> {IDENTIFIER} { return myTokenTypes.tagValueToken(); }
<DOC_TAG_VALUE_IN_LTGT> [\>] { yybegin(COMMENT_DATA); return myTokenTypes.tagValueGT(); }
<COMMENT_DATA_START, COMMENT_DATA> "{" {
if (checkAhead('@')){
if (checkAhead('@')) {
yybegin(INLINE_TAG_NAME);
return myTokenTypes.inlineTagStart();
}
@@ -89,13 +99,12 @@ IDENTIFIER={ALPHA}({ALPHA}|{DIGIT}|[":.-"])*
<COMMENT_DATA_START, COMMENT_DATA, DOC_TAG_VALUE> . { yybegin(COMMENT_DATA); return myTokenTypes.commentData(); }
<COMMENT_DATA_START> "@"{IDENTIFIER} { yybegin(TAG_DOC_SPACE); return myTokenTypes.tagName(); }
<TAG_DOC_SPACE> {WHITE_DOC_SPACE_CHAR}+ {
if (checkAhead('<') || checkAhead('\"')) yybegin(COMMENT_DATA);
else if (checkAhead('\u007b') ) yybegin(COMMENT_DATA); //lbrace - there's some error in JLex when typing lbrace directly
else if (checkAhead('\u007b') ) yybegin(COMMENT_DATA); // lbrace - there's a error in JLex when typing lbrace directly
else yybegin(DOC_TAG_VALUE);
return myTokenTypes.space();
return myTokenTypes.space();
}
"*"+"/" { return myTokenTypes.commentEnd(); }
java/java-psi-impl/src/com/intellij/lang/java/lexer/_JavaDocLexer.java
+721
View File
@@ -0,0 +1,721 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* The following code was generated by JFlex 1.4.3 on 05.06.13 19:07 */
/* It's an automatically generated code. Do not modify it. */
package com.intellij.lang.java.lexer;
import com.intellij.lexer.DocCommentTokenTypes;
import com.intellij.lexer.FlexLexer;
import com.intellij.psi.tree.IElementType;
@SuppressWarnings("ALL")
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 05.06.13 19:07 from the specification file
* <tt>C:/Users/roman.shevchenko/Projects/IDEA/tools/lexer/../../community/java/java-psi-impl/src/com/intellij/lang/java/lexer/_JavaDocLexer.flex</tt>
*/
class _JavaDocLexer implements FlexLexer {
/** initial size of the lookahead buffer */
private static final int ZZ_BUFFERSIZE = 16384;
/** lexical states */
public static final int INLINE_TAG_NAME = 16;
public static final int PARAM_TAG_SPACE = 8;
public static final int COMMENT_DATA = 4;
public static final int DOC_TAG_VALUE = 10;
public static final int COMMENT_DATA_START = 2;
public static final int TAG_DOC_SPACE = 6;
public static final int YYINITIAL = 0;
public static final int DOC_TAG_VALUE_IN_PAREN = 12;
public static final int DOC_TAG_VALUE_IN_LTGT = 14;
/**
* ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
* ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
* at the beginning of a line
* l is of the form l = 2*k, k a non negative integer
*/
private static final int ZZ_LEXSTATE[] = {
0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
8, 8
};
/**
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
"\11\0\1\2\1\1\1\0\1\2\1\1\22\0\1\2\2\0\1\13"+
"\1\4\3\0\1\11\1\12\1\7\1\0\1\14\1\5\1\3\1\6"+
"\12\3\1\5\1\0\1\22\1\0\1\23\1\0\1\15\32\4\1\10"+
"\1\0\1\10\1\0\1\4\1\0\1\17\13\4\1\21\2\4\1\16"+
"\1\4\1\20\10\4\1\24\1\0\1\25\44\0\4\4\4\0\1\4"+
"\12\0\1\4\4\0\1\4\5\0\27\4\1\0\37\4\1\0\u01ca\4"+
"\4\0\14\4\16\0\5\4\7\0\1\4\1\0\1\4\201\0\5\4"+
"\1\0\2\4\2\0\4\4\10\0\1\4\1\0\3\4\1\0\1\4"+
"\1\0\24\4\1\0\123\4\1\0\213\4\10\0\236\4\11\0\46\4"+
"\2\0\1\4\7\0\47\4\110\0\33\4\5\0\3\4\30\0\1\4"+
"\24\0\53\4\43\0\2\4\1\0\143\4\1\0\1\4\17\0\2\4"+
"\7\0\2\4\12\0\3\4\2\0\1\4\20\0\1\4\1\0\36\4"+
"\35\0\131\4\13\0\1\4\30\0\41\4\11\0\2\4\4\0\1\4"+
"\5\0\26\4\4\0\1\4\11\0\1\4\3\0\1\4\27\0\31\4"+
"\253\0\66\4\3\0\1\4\22\0\1\4\7\0\12\4\17\0\7\4"+
"\1\0\7\4\5\0\10\4\2\0\2\4\2\0\26\4\1\0\7\4"+
"\1\0\1\4\3\0\4\4\3\0\1\4\20\0\1\4\15\0\2\4"+
"\1\0\3\4\16\0\4\4\7\0\1\4\11\0\6\4\4\0\2\4"+
"\2\0\26\4\1\0\7\4\1\0\2\4\1\0\2\4\1\0\2\4"+
"\37\0\4\4\1\0\1\4\23\0\3\4\20\0\11\4\1\0\3\4"+
"\1\0\26\4\1\0\7\4\1\0\2\4\1\0\5\4\3\0\1\4"+
"\22\0\1\4\17\0\2\4\17\0\1\4\23\0\10\4\2\0\2\4"+
"\2\0\26\4\1\0\7\4\1\0\2\4\1\0\5\4\3\0\1\4"+
"\36\0\2\4\1\0\3\4\17\0\1\4\21\0\1\4\1\0\6\4"+
"\3\0\3\4\1\0\4\4\3\0\2\4\1\0\1\4\1\0\2\4"+
"\3\0\2\4\3\0\3\4\3\0\14\4\26\0\1\4\50\0\1\4"+
"\13\0\10\4\1\0\3\4\1\0\27\4\1\0\12\4\1\0\5\4"+
"\3\0\1\4\32\0\2\4\6\0\2\4\43\0\10\4\1\0\3\4"+
"\1\0\27\4\1\0\12\4\1\0\5\4\3\0\1\4\40\0\1\4"+
"\1\0\2\4\17\0\2\4\22\0\10\4\1\0\3\4\1\0\51\4"+
"\2\0\1\4\20\0\1\4\21\0\2\4\30\0\6\4\5\0\22\4"+
"\3\0\30\4\1\0\11\4\1\0\1\4\2\0\7\4\72\0\60\4"+
"\1\0\2\4\13\0\10\4\72\0\2\4\1\0\1\4\2\0\2\4"+
"\1\0\1\4\2\0\1\4\6\0\4\4\1\0\7\4\1\0\3\4"+
"\1\0\1\4\1\0\1\4\2\0\2\4\1\0\4\4\1\0\2\4"+
"\11\0\1\4\2\0\5\4\1\0\1\4\25\0\2\4\42\0\1\4"+
"\77\0\10\4\1\0\44\4\33\0\5\4\163\0\53\4\24\0\1\4"+
"\20\0\6\4\4\0\4\4\3\0\1\4\3\0\2\4\7\0\3\4"+
"\4\0\15\4\14\0\1\4\21\0\46\4\12\0\53\4\1\0\1\4"+
"\3\0\u0149\4\1\0\4\4\2\0\7\4\1\0\1\4\1\0\4\4"+
"\2\0\51\4\1\0\4\4\2\0\41\4\1\0\4\4\2\0\7\4"+
"\1\0\1\4\1\0\4\4\2\0\17\4\1\0\71\4\1\0\4\4"+
"\2\0\103\4\45\0\20\4\20\0\125\4\14\0\u026c\4\2\0\21\4"+
"\1\0\32\4\5\0\113\4\3\0\3\4\17\0\15\4\1\0\4\4"+
"\16\0\22\4\16\0\22\4\16\0\15\4\1\0\3\4\17\0\64\4"+
"\43\0\1\4\3\0\2\4\103\0\130\4\10\0\51\4\1\0\1\4"+
"\5\0\106\4\12\0\35\4\63\0\36\4\2\0\5\4\13\0\54\4"+
"\25\0\7\4\70\0\27\4\11\0\65\4\122\0\1\4\135\0\57\4"+
"\21\0\7\4\67\0\36\4\15\0\2\4\20\0\46\4\32\0\44\4"+
"\51\0\3\4\12\0\44\4\153\0\4\4\1\0\4\4\16\0\300\4"+
"\100\0\u0116\4\2\0\6\4\2\0\46\4\2\0\6\4\2\0\10\4"+
"\1\0\1\4\1\0\1\4\1\0\1\4\1\0\37\4\2\0\65\4"+
"\1\0\7\4\1\0\1\4\3\0\3\4\1\0\7\4\3\0\4\4"+
"\2\0\6\4\4\0\15\4\5\0\3\4\1\0\7\4\102\0\2\4"+
"\23\0\1\4\34\0\1\4\15\0\1\4\20\0\15\4\3\0\32\4"+
"\110\0\1\4\4\0\1\4\2\0\12\4\1\0\1\4\3\0\5\4"+
"\6\0\1\4\1\0\1\4\1\0\1\4\1\0\4\4\1\0\13\4"+
"\2\0\4\4\5\0\5\4\4\0\1\4\21\0\51\4\u0a77\0\57\4"+
"\1\0\57\4\1\0\205\4\6\0\4\4\21\0\46\4\12\0\66\4"+
"\11\0\1\4\20\0\27\4\11\0\7\4\1\0\7\4\1\0\7\4"+
"\1\0\7\4\1\0\7\4\1\0\7\4\1\0\7\4\1\0\7\4"+
"\120\0\1\4\u01d5\0\3\4\31\0\11\4\7\0\5\4\2\0\5\4"+
"\4\0\126\4\6\0\3\4\1\0\132\4\1\0\4\4\5\0\51\4"+
"\3\0\136\4\21\0\33\4\65\0\20\4\u0200\0\u19b6\4\112\0\u51cc\4"+
"\64\0\u048d\4\103\0\56\4\2\0\u010d\4\3\0\20\4\12\0\2\4"+
"\24\0\57\4\20\0\31\4\10\0\120\4\47\0\11\4\2\0\147\4"+
"\2\0\4\4\1\0\2\4\16\0\12\4\120\0\10\4\1\0\3\4"+
"\1\0\4\4\1\0\27\4\25\0\1\4\7\0\64\4\16\0\62\4"+
"\76\0\6\4\3\0\1\4\16\0\34\4\12\0\27\4\31\0\35\4"+
"\7\0\57\4\34\0\1\4\60\0\51\4\27\0\3\4\1\0\10\4"+
"\24\0\27\4\3\0\1\4\5\0\60\4\1\0\1\4\3\0\2\4"+
"\2\0\5\4\2\0\1\4\1\0\1\4\30\0\3\4\43\0\6\4"+
"\2\0\6\4\2\0\6\4\11\0\7\4\1\0\7\4\221\0\43\4"+
"\35\0\u2ba4\4\14\0\27\4\4\0\61\4\u2104\0\u012e\4\2\0\76\4"+
"\2\0\152\4\46\0\7\4\14\0\5\4\5\0\1\4\1\0\12\4"+
"\1\0\15\4\1\0\5\4\1\0\1\4\1\0\2\4\1\0\2\4"+
"\1\0\154\4\41\0\u016b\4\22\0\100\4\2\0\66\4\50\0\15\4"+
"\66\0\2\4\30\0\3\4\31\0\1\4\6\0\5\4\1\0\207\4"+
"\7\0\1\4\34\0\32\4\4\0\1\4\1\0\32\4\13\0\131\4"+
"\3\0\6\4\2\0\6\4\2\0\6\4\2\0\3\4\3\0\2\4"+
"\3\0\2\4\31\0";
/**
* Translates characters to character classes
*/
private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
/**
* Translates DFA states to action switch labels.
*/
private static final int [] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
"\11\0\3\1\1\2\1\3\2\2\1\4\1\5\1\6"+
"\1\7\1\10\1\11\1\12\1\13\1\14\1\15\1\16"+
"\1\17\1\12\1\20\1\1\1\0\1\21\1\0\2\22"+
"\1\23\3\22\1\24";
private static int [] zzUnpackAction() {
int [] result = new int[41];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
}
private static int zzUnpackAction(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
do result[j++] = value; while (--count > 0);
}
return j;
}
/**
* Translates a state to a row index in the transition table
*/
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
"\0\0\0\26\0\54\0\102\0\130\0\156\0\204\0\232"+
"\0\260\0\306\0\334\0\362\0\306\0\u0108\0\362\0\u011e"+
"\0\306\0\306\0\u0134\0\u014a\0\u0160\0\u0176\0\u018c\0\306"+
"\0\306\0\306\0\306\0\306\0\u01a2\0\306\0\u011e\0\u01b8"+
"\0\306\0\362\0\u01ce\0\u01e4\0\306\0\u01fa\0\u0210\0\u0226"+
"\0\u01ce";
private static int [] zzUnpackRowMap() {
int [] result = new int[41];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
}
private static int zzUnpackRowMap(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int high = packed.charAt(i++) << 16;
result[j++] = high | packed.charAt(i++);
}
return j;
}
/**
* The transition table of the DFA
*/
private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
"\6\12\1\13\1\14\16\12\1\15\2\16\4\15\1\17"+
"\5\15\1\20\6\15\1\21\1\22\1\15\1\16\1\23"+
"\4\15\1\17\14\15\1\21\1\22\1\12\2\24\4\12"+
"\1\14\15\12\1\22\1\12\2\25\4\12\1\14\16\12"+
"\1\15\2\26\2\27\2\15\1\17\1\27\1\30\1\15"+
"\1\31\1\32\1\15\4\27\1\33\2\15\1\22\1\12"+
"\2\16\2\27\2\12\1\14\1\27\1\12\1\34\1\12"+
"\1\32\1\12\4\27\10\12\1\35\2\12\1\14\6\12"+
"\4\35\1\12\1\36\11\12\1\14\5\12\1\37\10\12"+
"\35\0\1\40\24\0\1\41\1\42\17\0\2\16\27\0"+
"\1\43\11\0\1\44\3\43\6\0\1\23\24\0\2\24"+
"\24\0\2\25\24\0\2\26\26\0\2\27\3\0\1\27"+
"\5\0\4\27\7\0\3\35\10\0\4\35\13\0\1\45"+
"\21\0\3\43\10\0\4\43\7\0\3\43\10\0\1\43"+
"\1\46\2\43\7\0\3\43\10\0\2\43\1\47\1\43"+
"\7\0\3\43\10\0\1\43\1\50\2\43\7\0\3\43"+
"\10\0\3\43\1\51\4\0";
private static int [] zzUnpackTrans() {
int [] result = new int[572];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
}
private static int zzUnpackTrans(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
value--;
do result[j++] = value; while (--count > 0);
}
return j;
}
/* error codes */
private static final int ZZ_UNKNOWN_ERROR = 0;
private static final int ZZ_NO_MATCH = 1;
private static final int ZZ_PUSHBACK_2BIG = 2;
private static final char[] EMPTY_BUFFER = new char[0];
private static final int YYEOF = -1;
private static java.io.Reader zzReader = null; // Fake
/* error messages for the codes above */
private static final String ZZ_ERROR_MSG[] = {
"Unkown internal scanner error",
"Error: could not match input",
"Error: pushback value was too large"
};
/**
* ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
*/
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
"\11\0\1\11\2\1\1\11\3\1\2\11\5\1\5\11"+
"\1\1\1\11\1\1\1\0\1\11\1\0\2\1\1\11"+
"\4\1";
private static int [] zzUnpackAttribute() {
int [] result = new int[41];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
}
private static int zzUnpackAttribute(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
do result[j++] = value; while (--count > 0);
}
return j;
}
/** the current state of the DFA */
private int zzState;
/** the current lexical state */
private int zzLexicalState = YYINITIAL;
/** this buffer contains the current text to be matched and is
the source of the yytext() string */
private CharSequence zzBuffer = "";
/** this buffer may contains the current text array to be matched when it is cheap to acquire it */
private char[] zzBufferArray;
/** the textposition at the last accepting state */
private int zzMarkedPos;
/** the textposition at the last state to be included in yytext */
private int zzPushbackPos;
/** the current text position in the buffer */
private int zzCurrentPos;
/** startRead marks the beginning of the yytext() string in the buffer */
private int zzStartRead;
/** endRead marks the last character in the buffer, that has been read
from input */
private int zzEndRead;
/**
* zzAtBOL == true <=> the scanner is currently at the beginning of a line
*/
private boolean zzAtBOL = true;
/** zzAtEOF == true <=> the scanner is at the EOF */
private boolean zzAtEOF;
/** denotes if the user-EOF-code has already been executed */
private boolean zzEOFDone;
/* user code: */
private boolean myJdk15Enabled;
private DocCommentTokenTypes myTokenTypes;
public _JavaDocLexer(boolean isJdk15Enabled, DocCommentTokenTypes tokenTypes) {
this((java.io.Reader)null);
myJdk15Enabled = isJdk15Enabled;
myTokenTypes = tokenTypes;
}
public boolean checkAhead(char c) {
if (zzMarkedPos >= zzBuffer.length()) return false;
return zzBuffer.charAt(zzMarkedPos) == c;
}
public void goTo(int offset) {
zzCurrentPos = zzMarkedPos = zzStartRead = offset;
zzPushbackPos = 0;
zzAtEOF = offset < zzEndRead;
}
_JavaDocLexer(java.io.Reader in) {
this.zzReader = in;
}
/**
* Creates a new scanner.
* There is also java.io.Reader version of this constructor.
*
* @param in the java.io.Inputstream to read input from.
*/
_JavaDocLexer(java.io.InputStream in) {
this(new java.io.InputStreamReader(in));
}
/**
* Unpacks the compressed character translation table.
*
* @param packed the packed character translation table
* @return the unpacked character translation table
*/
private static char [] zzUnpackCMap(String packed) {
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 1606) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
}
return map;
}
public final int getTokenStart(){
return zzStartRead;
}
public final int getTokenEnd(){
return getTokenStart() + yylength();
}
public void reset(CharSequence buffer, int start, int end,int initialState){
zzBuffer = buffer;
zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer);
zzCurrentPos = zzMarkedPos = zzStartRead = start;
zzPushbackPos = 0;
zzAtEOF = false;
zzAtBOL = true;
zzEndRead = end;
yybegin(initialState);
}
/**
* Refills the input buffer.
*
* @return <code>false</code>, iff there was new input.
*
* @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
return true;
}
/**
* Returns the current lexical state.
*/
public final int yystate() {
return zzLexicalState;
}
/**
* Enters a new lexical state
*
* @param newState the new lexical state
*/
public final void yybegin(int newState) {
zzLexicalState = newState;
}
/**
* Returns the text matched by the current regular expression.
*/
public final CharSequence yytext() {
return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
}
/**
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* It is equivalent to yytext().charAt(pos), but faster
*
* @param pos the position of the character to fetch.
* A value from 0 to yylength()-1.
*
* @return the character at position pos
*/
public final char yycharat(int pos) {
return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos);
}
/**
* Returns the length of the matched text region.
*/
public final int yylength() {
return zzMarkedPos-zzStartRead;
}
/**
* Reports an error that occured while scanning.
*
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* will only be called with things that "Can't Possibly Happen".
* If this method is called, something is seriously wrong
* (e.g. a JFlex bug producing a faulty scanner etc.).
*
* Usual syntax/scanner level error handling should be done
* in error fallback rules.
*
* @param errorCode the code of the errormessage to display
*/
private void zzScanError(int errorCode) {
String message;
try {
message = ZZ_ERROR_MSG[errorCode];
}
catch (ArrayIndexOutOfBoundsException e) {
message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
}
throw new Error(message);
}
/**
* Pushes the specified amount of characters back into the input stream.
*
* They will be read again by then next call of the scanning method
*
* @param number the number of characters to be read again.
* This number must not be greater than yylength()!
*/
public void yypushback(int number) {
if ( number > yylength() )
zzScanError(ZZ_PUSHBACK_2BIG);
zzMarkedPos -= number;
}
/**
* Contains user EOF-code, which will be executed exactly once,
* when the end of file is reached
*/
private void zzDoEOF() {
if (!zzEOFDone) {
zzEOFDone = true;
}
}
/**
* Resumes scanning until the next regular expression is matched,
* the end of input is encountered or an I/O-Error occurs.
*
* @return the next token
* @exception java.io.IOException if any I/O-Error occurs
*/
public IElementType advance() throws java.io.IOException {
int zzInput;
int zzAction;
// cached fields:
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
CharSequence zzBufferL = zzBuffer;
char[] zzBufferArrayL = zzBufferArray;
char [] zzCMapL = ZZ_CMAP;
int [] zzTransL = ZZ_TRANS;
int [] zzRowMapL = ZZ_ROWMAP;
int [] zzAttrL = ZZ_ATTRIBUTE;
while (true) {
zzMarkedPosL = zzMarkedPos;
zzAction = -1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
zzState = ZZ_LEXSTATE[zzLexicalState];
zzForAction: {
while (true) {
if (zzCurrentPosL < zzEndReadL)
zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++));
else if (zzAtEOF) {
zzInput = YYEOF;
break zzForAction;
}
else {
// store back cached positions
zzCurrentPos = zzCurrentPosL;
zzMarkedPos = zzMarkedPosL;
boolean eof = zzRefill();
// get translated positions and possibly new buffer
zzCurrentPosL = zzCurrentPos;
zzMarkedPosL = zzMarkedPos;
zzBufferL = zzBuffer;
zzEndReadL = zzEndRead;
if (eof) {
zzInput = YYEOF;
break zzForAction;
}
else {
zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++));
}
}
int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
if (zzNext == -1) break zzForAction;
zzState = zzNext;
int zzAttributes = zzAttrL[zzState];
if ( (zzAttributes & 1) == 1 ) {
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
if ( (zzAttributes & 8) == 8 ) break zzForAction;
}
}
}
// store back cached position
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
case 12:
{ return myTokenTypes.tagValueSharp();
}
case 21: break;
case 16:
{ yybegin(COMMENT_DATA); return myTokenTypes.tagValueGT();
}
case 22: break;
case 3:
{ return myTokenTypes.space();
}
case 23: break;
case 6:
{ return myTokenTypes.commentData();
}
case 24: break;
case 5:
{ yybegin(COMMENT_DATA); return myTokenTypes.inlineTagEnd();
}
case 25: break;
case 19:
{ yybegin(COMMENT_DATA_START); return myTokenTypes.commentStart();
}
case 26: break;
case 7:
{ if (checkAhead('<') || checkAhead('\"')) yybegin(COMMENT_DATA);
else if (checkAhead('\u007b') ) yybegin(COMMENT_DATA); // lbrace - there's a error in JLex when typing lbrace directly
else yybegin(DOC_TAG_VALUE);
return myTokenTypes.space();
}
case 27: break;
case 8:
{ yybegin(DOC_TAG_VALUE); return myTokenTypes.space();
}
case 28: break;
case 10:
{ return myTokenTypes.tagValueToken();
}
case 29: break;
case 18:
{ yybegin(TAG_DOC_SPACE); return myTokenTypes.tagName();
}
case 30: break;
case 9:
{ yybegin(COMMENT_DATA); return myTokenTypes.space();
}
case 31: break;
case 20:
{ yybegin(PARAM_TAG_SPACE); return myTokenTypes.tagName();
}
case 32: break;
case 1:
{ return myTokenTypes.badCharacter();
}
case 33: break;
case 15:
{ yybegin(DOC_TAG_VALUE); return myTokenTypes.tagValueRParen();
}
case 34: break;
case 11:
{ yybegin(DOC_TAG_VALUE_IN_PAREN); return myTokenTypes.tagValueLParen();
}
case 35: break;
case 13:
{ return myTokenTypes.tagValueComma();
}
case 36: break;
case 14:
{ if (myJdk15Enabled) {
yybegin(DOC_TAG_VALUE_IN_LTGT);
return myTokenTypes.tagValueLT();
}
else {
yybegin(COMMENT_DATA);
return myTokenTypes.commentData();
}
}
case 37: break;
case 4:
{ if (checkAhead('@')) {
yybegin(INLINE_TAG_NAME);
return myTokenTypes.inlineTagStart();
}
else{
yybegin(COMMENT_DATA);
return myTokenTypes.inlineTagStart();
}
}
case 38: break;
case 2:
{ yybegin(COMMENT_DATA); return myTokenTypes.commentData();
}
case 39: break;
case 17:
{ return myTokenTypes.commentEnd();
}
case 40: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
zzDoEOF();
return null;
}
else {
zzScanError(ZZ_NO_MATCH);
}
}
}
}
}
java/java-psi-impl/src/com/intellij/lexer/_JavaLexer.flex → java/java-psi-impl/src/com/intellij/lang/java/lexer/_JavaLexer.flex
+1 -1
View File
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lexer;
package com.intellij.lang.java.lexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
java/java-psi-impl/src/com/intellij/lexer/_JavaLexer.java → java/java-psi-impl/src/com/intellij/lang/java/lexer/_JavaLexer.java
+108 -85
View File
@@ -14,9 +14,10 @@
* limitations under the License.
*/
/* The following code was generated by JFlex 1.4.3 on 1/17/13 8:47 PM */
package com.intellij.lexer;
/* The following code was generated by JFlex 1.4.3 on 05.06.13 17:47 */
package com.intellij.lang.java.lexer;
import com.intellij.lexer.FlexLexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.impl.source.tree.JavaDocElementType;
@@ -27,8 +28,8 @@ import com.intellij.psi.tree.IElementType;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 1/17/13 8:47 PM from the specification file
* <tt>/home/sher/Projects/IDEA/tools/lexer/../../community/java/java-psi-impl/src/com/intellij/lexer/_JavaLexer.flex</tt>
* on 05.06.13 17:47 from the specification file
* <tt>C:/Users/roman.shevchenko/Projects/IDEA/tools/lexer/../../community/java/java-psi-impl/src/com/intellij/lang/java/lexer/_JavaLexer.flex</tt>
*/
class _JavaLexer implements FlexLexer {
/** initial size of the lookahead buffer */
@@ -60,87 +61,109 @@ class _JavaLexer implements FlexLexer {
"\1\46\1\2\1\43\1\35\1\50\1\37\1\42\1\53\1\2\1\30"+
"\1\36\1\27\1\31\1\54\1\55\1\51\1\44\1\56\1\72\1\61"+
"\1\73\1\100\41\3\2\0\4\2\4\0\1\2\2\0\1\3\7\0"+
"\1\2\4\0\1\2\5\0\27\2\1\0\37\2\1\0\u013f\2\31\0"+
"\162\2\4\0\14\2\16\0\5\2\11\0\1\2\21\0\130\3\5\0"+
"\23\3\12\0\1\2\13\0\1\2\1\0\3\2\1\0\1\2\1\0"+
"\24\2\1\0\54\2\1\0\46\2\1\0\5\2\4\0\202\2\1\0"+
"\4\3\3\0\105\2\1\0\46\2\2\0\2\2\6\0\20\2\41\0"+
"\46\2\2\0\1\2\7\0\47\2\11\0\21\3\1\0\27\3\1\0"+
"\3\3\1\0\1\3\1\0\2\3\1\0\1\3\13\0\33\2\5\0"+
"\3\2\15\0\4\3\14\0\6\3\13\0\32\2\5\0\13\2\16\3"+
"\7\0\12\3\4\0\2\2\1\3\143\2\1\0\1\2\10\3\1\0"+
"\6\3\2\2\2\3\1\0\4\3\2\2\12\3\3\2\2\0\1\2"+
"\17\0\1\3\1\2\1\3\36\2\33\3\2\0\3\2\60\0\46\2"+
"\13\3\1\2\u014f\0\3\3\66\2\2\0\1\3\1\2\20\3\2\0"+
"\1\2\4\3\3\0\12\2\2\3\2\0\12\3\21\0\3\3\1\0"+
"\10\2\2\0\2\2\2\0\26\2\1\0\7\2\1\0\1\2\3\0"+
"\4\2\2\0\1\3\1\2\7\3\2\0\2\3\2\0\3\3\11\0"+
"\1\3\4\0\2\2\1\0\3\2\2\3\2\0\12\3\4\2\15\0"+
"\3\3\1\0\6\2\4\0\2\2\2\0\26\2\1\0\7\2\1\0"+
"\2\2\1\0\2\2\1\0\2\2\2\0\1\3\1\0\5\3\4\0"+
"\2\3\2\0\3\3\13\0\4\2\1\0\1\2\7\0\14\3\3\2"+
"\14\0\3\3\1\0\11\2\1\0\3\2\1\0\26\2\1\0\7\2"+
"\1\0\2\2\1\0\5\2\2\0\1\3\1\2\10\3\1\0\3\3"+
"\1\0\3\3\2\0\1\2\17\0\2\2\2\3\2\0\12\3\1\0"+
"\1\2\17\0\3\3\1\0\10\2\2\0\2\2\2\0\26\2\1\0"+
"\7\2\1\0\2\2\1\0\5\2\2\0\1\3\1\2\6\3\3\0"+
"\2\3\2\0\3\3\10\0\2\3\4\0\2\2\1\0\3\2\4\0"+
"\12\3\1\0\1\2\20\0\1\3\1\2\1\0\6\2\3\0\3\2"+
"\1\0\4\2\3\0\2\2\1\0\1\2\1\0\2\2\3\0\2\2"+
"\3\0\3\2\3\0\10\2\1\0\3\2\4\0\5\3\3\0\3\3"+
"\1\0\4\3\11\0\1\3\17\0\11\3\11\0\1\2\7\0\3\3"+
"\1\2\4\0\1\2\5\0\27\2\1\0\37\2\1\0\u01ca\2\4\0"+
"\14\2\16\0\5\2\7\0\1\2\1\0\1\2\21\0\160\3\5\2"+
"\1\0\2\2\2\0\4\2\10\0\1\2\1\0\3\2\1\0\1\2"+
"\1\0\24\2\1\0\123\2\1\0\213\2\1\0\5\3\2\0\236\2"+
"\11\0\46\2\2\0\1\2\7\0\47\2\11\0\55\3\1\0\1\3"+
"\1\0\2\3\1\0\2\3\1\0\1\3\10\0\33\2\5\0\3\2"+
"\15\0\4\3\7\0\1\2\4\0\13\3\5\0\53\2\37\3\4\0"+
"\2\2\1\3\143\2\1\0\1\2\10\3\1\0\6\3\2\2\2\3"+
"\1\0\4\3\2\2\12\3\3\2\2\0\1\2\17\0\1\3\1\2"+
"\1\3\36\2\33\3\2\0\131\2\13\3\1\2\16\0\12\3\41\2"+
"\11\3\2\2\4\0\1\2\5\0\26\2\4\3\1\2\11\3\1\2"+
"\3\3\1\2\5\3\22\0\31\2\3\3\244\0\4\3\66\2\3\3"+
"\1\2\22\3\1\2\7\3\12\2\2\3\2\0\12\3\1\0\7\2"+
"\1\0\7\2\1\0\3\3\1\0\10\2\2\0\2\2\2\0\26\2"+
"\1\0\7\2\1\0\1\2\3\0\4\2\2\0\1\3\1\2\7\3"+
"\2\0\2\3\2\0\3\3\1\2\10\0\1\3\4\0\2\2\1\0"+
"\3\2\2\3\2\0\12\3\4\2\7\0\1\2\5\0\3\3\1\0"+
"\6\2\4\0\2\2\2\0\26\2\1\0\7\2\1\0\2\2\1\0"+
"\2\2\1\0\2\2\2\0\1\3\1\0\5\3\4\0\2\3\2\0"+
"\3\3\3\0\1\3\7\0\4\2\1\0\1\2\7\0\14\3\3\2"+
"\1\3\13\0\3\3\1\0\11\2\1\0\3\2\1\0\26\2\1\0"+
"\7\2\1\0\2\2\1\0\5\2\2\0\1\3\1\2\10\3\1\0"+
"\3\3\1\0\3\3\2\0\1\2\17\0\2\2\2\3\2\0\12\3"+
"\1\0\1\2\17\0\3\3\1\0\10\2\2\0\2\2\2\0\26\2"+
"\1\0\7\2\1\0\2\2\1\0\5\2\2\0\1\3\1\2\7\3"+
"\2\0\2\3\2\0\3\3\10\0\2\3\4\0\2\2\1\0\3\2"+
"\2\3\2\0\12\3\1\0\1\2\20\0\1\3\1\2\1\0\6\2"+
"\3\0\3\2\1\0\4\2\3\0\2\2\1\0\1\2\1\0\2\2"+
"\3\0\2\2\3\0\3\2\3\0\14\2\4\0\5\3\3\0\3\3"+
"\1\0\4\3\2\0\1\2\6\0\1\3\16\0\12\3\11\0\1\2"+
"\7\0\3\3\1\0\10\2\1\0\3\2\1\0\27\2\1\0\12\2"+
"\1\0\5\2\3\0\1\2\7\3\1\0\3\3\1\0\4\3\7\0"+
"\2\3\1\0\2\2\6\0\2\2\2\3\2\0\12\3\22\0\2\3"+
"\1\0\10\2\1\0\3\2\1\0\27\2\1\0\12\2\1\0\5\2"+
"\4\0\7\3\1\0\3\3\1\0\4\3\7\0\2\3\11\0\2\2"+
"\4\0\12\3\22\0\2\3\1\0\10\2\1\0\3\2\1\0\27\2"+
"\1\0\12\2\1\0\5\2\2\0\1\3\1\2\7\3\1\0\3\3"+
"\1\0\4\3\7\0\2\3\7\0\1\2\1\0\2\2\4\0\12\3"+
"\22\0\2\3\1\0\10\2\1\0\3\2\1\0\27\2\1\0\20\2"+
"\4\0\6\3\2\0\3\3\1\0\4\3\11\0\1\3\10\0\2\2"+
"\4\0\12\3\22\0\2\3\1\0\22\2\3\0\30\2\1\0\11\2"+
"\1\0\1\2\2\0\7\2\3\0\1\3\4\0\6\3\1\0\1\3"+
"\1\0\10\3\22\0\2\3\15\0\60\2\1\3\2\2\7\3\4\0"+
"\10\2\10\3\1\0\12\3\47\0\2\2\1\0\1\2\2\0\2\2"+
"\1\0\1\2\2\0\1\2\6\0\4\2\1\0\7\2\1\0\3\2"+
"\1\0\1\2\1\0\1\2\2\0\2\2\1\0\4\2\1\3\2\2"+
"\6\3\1\0\2\3\1\2\2\0\5\2\1\0\1\2\1\0\6\3"+
"\2\0\12\3\2\0\2\2\42\0\1\2\27\0\2\3\6\0\12\3"+
"\13\0\1\3\1\0\1\3\1\0\1\3\4\0\2\3\10\2\1\0"+
"\42\2\6\0\24\3\1\0\2\3\4\2\4\0\10\3\1\0\44\3"+
"\11\0\1\3\71\0\42\2\1\0\5\2\1\0\2\2\1\0\7\3"+
"\3\0\4\3\6\0\12\3\6\0\6\2\4\3\106\0\46\2\12\0"+
"\51\2\7\0\132\2\5\0\104\2\5\0\122\2\6\0\7\2\1\0"+
"\77\2\1\0\1\2\1\0\4\2\2\0\7\2\1\0\1\2\1\0"+
"\4\2\2\0\47\2\1\0\1\2\1\0\4\2\2\0\37\2\1\0"+
"\1\2\1\0\4\2\2\0\7\2\1\0\1\2\1\0\4\2\2\0"+
"\7\2\1\0\7\2\1\0\27\2\1\0\37\2\1\0\1\2\1\0"+
"\4\2\2\0\7\2\1\0\47\2\1\0\23\2\16\0\11\3\56\0"+
"\125\2\14\0\u026c\2\2\0\10\2\12\0\32\2\5\0\113\2\3\0"+
"\3\2\17\0\15\2\1\0\4\2\3\3\13\0\22\2\3\3\13\0"+
"\22\2\2\3\14\0\15\2\1\0\3\2\1\0\2\3\14\0\64\2"+
"\40\3\3\0\1\2\3\0\2\2\1\3\2\0\12\3\41\0\3\3"+
"\2\0\12\3\6\0\130\2\10\0\51\2\1\3\126\0\35\2\3\0"+
"\14\3\4\0\14\3\12\0\12\3\36\2\2\0\5\2\u038b\0\154\2"+
"\224\0\234\2\4\0\132\2\6\0\26\2\2\0\6\2\2\0\46\2"+
"\2\0\6\2\2\0\10\2\1\0\1\2\1\0\1\2\1\0\1\2"+
"\1\0\37\2\2\0\65\2\1\0\7\2\1\0\1\2\3\0\3\2"+
"\1\0\7\2\3\0\4\2\2\0\6\2\4\0\15\2\5\0\3\2"+
"\1\0\7\2\17\0\4\3\32\0\5\3\20\0\2\2\23\0\1\2"+
"\13\0\4\3\6\0\6\3\1\0\1\2\15\0\1\2\40\0\22\2"+
"\36\0\15\3\4\0\1\3\3\0\6\3\27\0\1\2\4\0\1\2"+
"\2\0\12\2\1\0\1\2\3\0\5\2\6\0\1\2\1\0\1\2"+
"\1\0\1\2\1\0\4\2\1\0\3\2\1\0\7\2\3\0\3\2"+
"\5\0\5\2\26\0\44\2\u0e81\0\3\2\31\0\11\2\6\3\1\0"+
"\5\2\2\0\5\2\4\0\126\2\2\0\2\3\2\0\3\2\1\0"+
"\137\2\5\0\50\2\4\0\136\2\21\0\30\2\70\0\20\2\u0200\0"+
"\u19b6\2\112\0\u51a6\2\132\0\u048d\2\u0773\0\u2ba4\2\u215c\0\u012e\2\2\0"+
"\73\2\225\0\7\2\14\0\5\2\5\0\1\2\1\3\12\2\1\0"+
"\15\2\1\0\5\2\1\0\1\2\1\0\2\2\1\0\2\2\1\0"+
"\154\2\41\0\u016b\2\22\0\100\2\2\0\66\2\50\0\15\2\3\0"+
"\20\3\20\0\4\3\17\0\2\2\30\0\3\2\31\0\1\2\6\0"+
"\5\2\1\0\207\2\2\0\1\3\4\0\1\2\13\0\12\3\7\0"+
"\32\2\4\0\1\2\1\0\32\2\12\0\132\2\3\0\6\2\2\0"+
"\6\2\2\0\6\2\2\0\3\2\3\0\2\2\3\0\2\2\22\0"+
"\3\3\4\0";
"\2\0\1\3\1\2\7\3\1\0\3\3\1\0\4\3\7\0\2\3"+
"\7\0\1\2\1\0\2\2\2\3\2\0\12\3\1\0\2\2\17\0"+
"\2\3\1\0\10\2\1\0\3\2\1\0\51\2\2\0\1\2\7\3"+
"\1\0\3\3\1\0\4\3\1\2\10\0\1\3\10\0\2\2\2\3"+
"\2\0\12\3\12\0\6\2\2\0\2\3\1\0\22\2\3\0\30\2"+
"\1\0\11\2\1\0\1\2\2\0\7\2\3\0\1\3\4\0\6\3"+
"\1\0\1\3\1\0\10\3\22\0\2\3\15\0\60\2\1\3\2\2"+
"\7\3\4\0\10\2\10\3\1\0\12\3\47\0\2\2\1\0\1\2"+
"\2\0\2\2\1\0\1\2\2\0\1\2\6\0\4\2\1\0\7\2"+
"\1\0\3\2\1\0\1\2\1\0\1\2\2\0\2\2\1\0\4\2"+
"\1\3\2\2\6\3\1\0\2\3\1\2\2\0\5\2\1\0\1\2"+
"\1\0\6\3\2\0\12\3\2\0\2\2\42\0\1\2\27\0\2\3"+
"\6\0\12\3\13\0\1\3\1\0\1\3\1\0\1\3\4\0\2\3"+
"\10\2\1\0\44\2\4\0\24\3\1\0\2\3\5\2\13\3\1\0"+
"\44\3\11\0\1\3\71\0\53\2\24\3\1\2\12\3\6\0\6\2"+
"\4\3\4\2\3\3\1\2\3\3\2\2\7\3\3\2\4\3\15\2"+
"\14\3\1\2\17\3\2\0\46\2\12\0\53\2\1\0\1\2\3\0"+
"\u0149\2\1\0\4\2\2\0\7\2\1\0\1\2\1\0\4\2\2\0"+
"\51\2\1\0\4\2\2\0\41\2\1\0\4\2\2\0\7\2\1\0"+
"\1\2\1\0\4\2\2\0\17\2\1\0\71\2\1\0\4\2\2\0"+
"\103\2\2\0\3\3\40\0\20\2\20\0\125\2\14\0\u026c\2\2\0"+
"\21\2\1\0\32\2\5\0\113\2\3\0\3\2\17\0\15\2\1\0"+
"\4\2\3\3\13\0\22\2\3\3\13\0\22\2\2\3\14\0\15\2"+
"\1\0\3\2\1\0\2\3\14\0\64\2\40\3\3\0\1\2\3\0"+
"\2\2\1\3\2\0\12\3\41\0\3\3\2\0\12\3\6\0\130\2"+
"\10\0\51\2\1\3\1\2\5\0\106\2\12\0\35\2\3\0\14\3"+
"\4\0\14\3\12\0\12\3\36\2\2\0\5\2\13\0\54\2\4\0"+
"\21\3\7\2\2\3\6\0\12\3\46\0\27\2\5\3\4\0\65\2"+
"\12\3\1\0\35\3\2\0\13\3\6\0\12\3\15\0\1\2\130\0"+
"\5\3\57\2\21\3\7\2\4\0\12\3\21\0\11\3\14\0\3\3"+
"\36\2\12\3\3\0\2\2\12\3\6\0\46\2\16\3\14\0\44\2"+
"\24\3\10\0\12\3\3\0\3\2\12\3\44\2\122\0\3\3\1\0"+
"\25\3\4\2\1\3\4\2\1\3\15\0\300\2\47\3\25\0\4\3"+
"\u0116\2\2\0\6\2\2\0\46\2\2\0\6\2\2\0\10\2\1\0"+
"\1\2\1\0\1\2\1\0\1\2\1\0\37\2\2\0\65\2\1\0"+
"\7\2\1\0\1\2\3\0\3\2\1\0\7\2\3\0\4\2\2\0"+
"\6\2\4\0\15\2\5\0\3\2\1\0\7\2\16\0\5\3\32\0"+
"\5\3\20\0\2\2\23\0\1\2\13\0\5\3\5\0\6\3\1\0"+
"\1\2\15\0\1\2\20\0\15\2\3\0\32\2\26\0\15\3\4\0"+
"\1\3\3\0\14\3\21\0\1\2\4\0\1\2\2\0\12\2\1\0"+
"\1\2\3\0\5\2\6\0\1\2\1\0\1\2\1\0\1\2\1\0"+
"\4\2\1\0\13\2\2\0\4\2\5\0\5\2\4\0\1\2\21\0"+
"\51\2\u0a77\0\57\2\1\0\57\2\1\0\205\2\6\0\4\2\3\3"+
"\16\0\46\2\12\0\66\2\11\0\1\2\17\0\1\3\27\2\11\0"+
"\7\2\1\0\7\2\1\0\7\2\1\0\7\2\1\0\7\2\1\0"+
"\7\2\1\0\7\2\1\0\7\2\1\0\40\3\57\0\1\2\u01d5\0"+
"\3\2\31\0\11\2\6\3\1\0\5\2\2\0\5\2\4\0\126\2"+
"\2\0\2\3\2\0\3\2\1\0\132\2\1\0\4\2\5\0\51\2"+
"\3\0\136\2\21\0\33\2\65\0\20\2\u0200\0\u19b6\2\112\0\u51cc\2"+
"\64\0\u048d\2\103\0\56\2\2\0\u010d\2\3\0\20\2\12\3\2\2"+
"\24\0\57\2\1\3\14\0\2\3\1\0\31\2\10\0\120\2\2\3"+
"\45\0\11\2\2\0\147\2\2\0\4\2\1\0\2\2\16\0\12\2"+
"\120\0\10\2\1\3\3\2\1\3\4\2\1\3\27\2\5\3\20\0"+
"\1\2\7\0\64\2\14\0\2\3\62\2\21\3\13\0\12\3\6\0"+
"\22\3\6\2\3\0\1\2\4\0\12\3\34\2\10\3\2\0\27\2"+
"\15\3\14\0\35\2\3\0\4\3\57\2\16\3\16\0\1\2\12\3"+
"\46\0\51\2\16\3\11\0\3\2\1\3\10\2\2\3\2\0\12\3"+
"\6\0\27\2\3\0\1\2\1\3\4\0\60\2\1\3\1\2\3\3"+
"\2\2\2\3\5\2\2\3\1\2\1\3\1\2\30\0\3\2\43\0"+
"\6\2\2\0\6\2\2\0\6\2\11\0\7\2\1\0\7\2\221\0"+
"\43\2\10\3\1\0\2\3\2\0\12\3\6\0\u2ba4\2\14\0\27\2"+
"\4\0\61\2\u2104\0\u012e\2\2\0\76\2\2\0\152\2\46\0\7\2"+
"\14\0\5\2\5\0\1\2\1\3\12\2\1\0\15\2\1\0\5\2"+
"\1\0\1\2\1\0\2\2\1\0\2\2\1\0\154\2\41\0\u016b\2"+
"\22\0\100\2\2\0\66\2\50\0\15\2\3\0\20\3\20\0\7\3"+
"\14\0\2\2\30\0\3\2\31\0\1\2\6\0\5\2\1\0\207\2"+
"\2\0\1\3\4\0\1\2\13\0\12\3\7\0\32\2\4\0\1\2"+
"\1\0\32\2\13\0\131\2\3\0\6\2\2\0\6\2\2\0\6\2"+
"\2\0\3\2\3\0\2\2\3\0\2\2\22\0\3\3\4\0";
/**
* Translates characters to character classes
@@ -749,7 +772,7 @@ class _JavaLexer implements FlexLexer {
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 1784) {
while (i < 2238) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
java/java-psi-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java
+1 -3
View File
@@ -20,7 +20,6 @@ import com.intellij.lang.*;
import com.intellij.lang.impl.PsiBuilderAdapter;
import com.intellij.lang.java.JavaLanguage;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.JavaDocLexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Condition;
@@ -202,8 +201,7 @@ public class JavaParserUtil {
final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
final Lexer lexer = chameleon.getElementType() == JavaDocElementType.DOC_COMMENT
? new JavaDocLexer(level.isAtLeast(LanguageLevel.JDK_1_5))
: JavaParserDefinition.createLexer(level);
? JavaParserDefinition.createDocLexer(level) : JavaParserDefinition.createLexer(level);
final PsiBuilder builder = factory.createBuilder(project, chameleon, lexer, chameleon.getElementType().getLanguage(), chameleon.getChars());
setLanguageLevel(builder, level);
java/java-psi-impl/src/com/intellij/lexer/JavaLexer.java
+6 -466
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,473 +15,13 @@
*/
package com.intellij.lexer;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.TokenType;
import com.intellij.psi.impl.source.tree.JavaDocElementType;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.text.CharArrayUtil;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
import java.io.IOException;
import java.util.Set;
public class JavaLexer extends LexerBase {
private static final HashTable[] TABLES = new HashTable[]{
new HashTable(LanguageLevel.JDK_1_5),
new HashTable(LanguageLevel.JDK_1_4),
new HashTable(LanguageLevel.JDK_1_3)
};
private static HashTable getTable(final LanguageLevel level) {
for (HashTable table : TABLES) {
if (level.isAtLeast(table.myLevel)) {
return table;
}
}
throw new IllegalArgumentException("Unsupported level: " + level);
/** @deprecated use com.intellij.lang.java.JavaLexer (to remove in IDEA 14) */
@SuppressWarnings({"ClassNameSameAsAncestorName", "UnusedDeclaration"})
public class JavaLexer extends com.intellij.lang.java.lexer.JavaLexer {
public JavaLexer(@NotNull LanguageLevel level) {
super(level);
}
public static boolean isKeyword(String id, LanguageLevel level) {
return getTable(level).contains(id);
}
private final _JavaLexer myFlexLexer;
private final HashTable myTable;
private CharSequence myBuffer;
private char[] myBufferArray;
private int myBufferIndex;
private int myBufferEndOffset;
private int myTokenEndOffset; // positioned after the last symbol of the current token
private IElementType myTokenType;
public JavaLexer(@NotNull final LanguageLevel level) {
myFlexLexer = new _JavaLexer(level);
myTable = getTable(level);
}
private static final class HashTable {
private static final int NUM_ENTRIES = 999;
private static final Logger LOG = Logger.getInstance("com.intellij.Lexer.JavaLexer");
private final LanguageLevel myLevel;
private final char[][] myTable = new char[NUM_ENTRIES][];
private final IElementType[] myKeywords = new IElementType[NUM_ENTRIES];
private final Set<String> myKeywordsInSet = new THashSet<String>();
private void add(String s, IElementType tokenType) {
char[] chars = s.toCharArray();
int hashCode = chars[0] * 2;
for (int j = 1; j < chars.length; j++) {
hashCode += chars[j];
}
int modHashCode = hashCode % NUM_ENTRIES;
LOG.assertTrue(myTable[modHashCode] == null);
myTable[modHashCode] = chars;
myKeywords[modHashCode] = tokenType;
myKeywordsInSet.add(s);
}
public boolean contains(String s) {
return myKeywordsInSet.contains(s);
}
private boolean contains(int hashCode, final char[] bufferArray, final CharSequence buffer, int offset) {
int modHashCode = hashCode % NUM_ENTRIES;
final char[] kwd = myTable[modHashCode];
if (kwd == null) return false;
if (bufferArray != null) {
for (int j = 0; j < kwd.length; j++) {
if (bufferArray[j + offset] != kwd[j]) return false;
}
} else {
for (int j = 0; j < kwd.length; j++) {
if (buffer.charAt(j + offset) != kwd[j]) return false;
}
}
return true;
}
private IElementType getTokenType(int hashCode) {
return myKeywords[hashCode % NUM_ENTRIES];
}
@SuppressWarnings({"HardCodedStringLiteral"})
private HashTable(final LanguageLevel level) {
myLevel = level;
if (level.isAtLeast(LanguageLevel.JDK_1_4)) {
add("assert", JavaTokenType.ASSERT_KEYWORD);
if (level.isAtLeast(LanguageLevel.JDK_1_5)) {
add("enum", JavaTokenType.ENUM_KEYWORD);
}
}
add("abstract", JavaTokenType.ABSTRACT_KEYWORD);
add("default", JavaTokenType.DEFAULT_KEYWORD);
add("if", JavaTokenType.IF_KEYWORD);
add("private", JavaTokenType.PRIVATE_KEYWORD);
add("this", JavaTokenType.THIS_KEYWORD);
add("boolean", JavaTokenType.BOOLEAN_KEYWORD);
add("do", JavaTokenType.DO_KEYWORD);
add("implements", JavaTokenType.IMPLEMENTS_KEYWORD);
add("protected", JavaTokenType.PROTECTED_KEYWORD);
add("throw", JavaTokenType.THROW_KEYWORD);
add("break", JavaTokenType.BREAK_KEYWORD);
add("double", JavaTokenType.DOUBLE_KEYWORD);
add("import", JavaTokenType.IMPORT_KEYWORD);
add("public", JavaTokenType.PUBLIC_KEYWORD);
add("throws", JavaTokenType.THROWS_KEYWORD);
add("byte", JavaTokenType.BYTE_KEYWORD);
add("else", JavaTokenType.ELSE_KEYWORD);
add("instanceof", JavaTokenType.INSTANCEOF_KEYWORD);
add("return", JavaTokenType.RETURN_KEYWORD);
add("transient", JavaTokenType.TRANSIENT_KEYWORD);
add("case", JavaTokenType.CASE_KEYWORD);
add("extends", JavaTokenType.EXTENDS_KEYWORD);
add("int", JavaTokenType.INT_KEYWORD);
add("short", JavaTokenType.SHORT_KEYWORD);
add("try", JavaTokenType.TRY_KEYWORD);
add("catch", JavaTokenType.CATCH_KEYWORD);
add("final", JavaTokenType.FINAL_KEYWORD);
add("interface", JavaTokenType.INTERFACE_KEYWORD);
add("static", JavaTokenType.STATIC_KEYWORD);
add("void", JavaTokenType.VOID_KEYWORD);
add("char", JavaTokenType.CHAR_KEYWORD);
add("finally", JavaTokenType.FINALLY_KEYWORD);
add("long", JavaTokenType.LONG_KEYWORD);
add("strictfp", JavaTokenType.STRICTFP_KEYWORD);
add("volatile", JavaTokenType.VOLATILE_KEYWORD);
add("class", JavaTokenType.CLASS_KEYWORD);
add("float", JavaTokenType.FLOAT_KEYWORD);
add("native", JavaTokenType.NATIVE_KEYWORD);
add("super", JavaTokenType.SUPER_KEYWORD);
add("while", JavaTokenType.WHILE_KEYWORD);
add("const", JavaTokenType.CONST_KEYWORD);
add("for", JavaTokenType.FOR_KEYWORD);
add("new", JavaTokenType.NEW_KEYWORD);
add("switch", JavaTokenType.SWITCH_KEYWORD);
add("continue", JavaTokenType.CONTINUE_KEYWORD);
add("goto", JavaTokenType.GOTO_KEYWORD);
add("package", JavaTokenType.PACKAGE_KEYWORD);
add("synchronized", JavaTokenType.SYNCHRONIZED_KEYWORD);
add("true", JavaTokenType.TRUE_KEYWORD);
add("false", JavaTokenType.FALSE_KEYWORD);
add("null", JavaTokenType.NULL_KEYWORD);
}
}
@Override
public final void start(CharSequence buffer, int startOffset, int endOffset, int initialState) {
myBuffer = buffer;
myBufferArray = CharArrayUtil.fromSequenceWithoutCopying(buffer);
myBufferIndex = startOffset;
myBufferEndOffset = endOffset;
myTokenType = null;
myTokenEndOffset = startOffset;
myFlexLexer.reset(myBuffer, startOffset, endOffset, 0);
}
@Override
public int getState() {
return 0;
}
@Override
public final IElementType getTokenType() {
if (myTokenType == null) _locateToken();
return myTokenType;
}
@Override
public final int getTokenStart() {
return myBufferIndex;
}
@Override
public final int getTokenEnd() {
if (myTokenType == null) _locateToken();
return myTokenEndOffset;
}
@Override
public final void advance() {
if (myTokenType == null) _locateToken();
myTokenType = null;
}
private void _locateToken() {
if (myTokenEndOffset == myBufferEndOffset) {
myTokenType = null;
myBufferIndex = myBufferEndOffset;
return;
}
myBufferIndex = myTokenEndOffset;
final char c = myBufferArray != null ? myBufferArray[myBufferIndex]:myBuffer.charAt(myBufferIndex);
switch (c) {
default:
flexLocateToken();
break;
case ' ':
case '\t':
case '\n':
case '\r':
case '\f':
myTokenType = TokenType.WHITE_SPACE;
myTokenEndOffset = getWhitespaces(myBufferIndex + 1);
break;
case '/':
if (myBufferIndex + 1 >= myBufferEndOffset) {
myTokenType = JavaTokenType.DIV;
myTokenEndOffset = myBufferEndOffset;
}
else {
final char nextChar = myBufferArray != null ? myBufferArray[myBufferIndex + 1]:myBuffer.charAt(myBufferIndex + 1);
if (nextChar == '/') {
myTokenType = JavaTokenType.END_OF_LINE_COMMENT;
myTokenEndOffset = getLineTerminator(myBufferIndex + 2);
}
else if (nextChar == '*') {
if (myBufferIndex + 2 >= myBufferEndOffset ||
(myBufferArray != null ? myBufferArray[myBufferIndex + 2]:myBuffer.charAt(myBufferIndex + 2)) != '*' ||
(myBufferIndex + 3 < myBufferEndOffset &&
(myBufferArray != null ? myBufferArray[myBufferIndex + 3]:myBuffer.charAt(myBufferIndex + 3)) == '/')) {
myTokenType = JavaTokenType.C_STYLE_COMMENT;
myTokenEndOffset = getClosingComment(myBufferIndex + 2);
}
else {
myTokenType = JavaDocElementType.DOC_COMMENT;
myTokenEndOffset = getClosingComment(myBufferIndex + 3);
}
}
else if (c > 127 && Character.isJavaIdentifierStart(c)) {
myTokenEndOffset = getIdentifier(myBufferIndex + 1);
}
else {
flexLocateToken();
}
}
break;
case '"':
case '\'':
myTokenType = c == '"' ? JavaTokenType.STRING_LITERAL : JavaTokenType.CHARACTER_LITERAL;
myTokenEndOffset = getClosingParenthesis(myBufferIndex + 1, c);
}
if (myTokenEndOffset > myBufferEndOffset) {
myTokenEndOffset = myBufferEndOffset;
}
}
private int getWhitespaces(int pos) {
if (pos >= myBufferEndOffset) return myBufferEndOffset;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
while (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f') {
pos++;
if (pos == myBufferEndOffset) return pos;
c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
return pos;
}
private void flexLocateToken() {
try {
myFlexLexer.goTo(myBufferIndex);
myTokenType = myFlexLexer.advance();
myTokenEndOffset = myFlexLexer.getTokenEnd();
}
catch (IOException e) {
// Can't be
}
}
private int getClosingParenthesis(int offset, char c) {
int pos = offset;
final int lBufferEnd = myBufferEndOffset;
if (pos >= lBufferEnd) return lBufferEnd;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
char cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
while (true) {
while (cur != c && cur != '\n' && cur != '\r' && cur != '\\') {
pos++;
if (pos >= lBufferEnd) return lBufferEnd;
cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
if (cur == '\\') {
pos++;
if (pos >= lBufferEnd) return lBufferEnd;
cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
if (cur == '\n' || cur == '\r') continue;
pos++;
if (pos >= lBufferEnd) return lBufferEnd;
cur = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
else if (cur == c) {
break;
}
else {
pos--;
break;
}
}
return pos + 1;
}
private int getClosingComment(int offset) {
int pos = offset;
final int lBufferEnd = myBufferEndOffset;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
while (pos < lBufferEnd - 1) {
final char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
if (c == '*' && (lBufferArray != null ? lBufferArray[pos + 1]:lBuffer.charAt(pos + 1)) == '/') {
break;
}
pos++;
}
return pos + 2;
}
private int getLineTerminator(int offset) {
int pos = offset;
final int lBufferEnd = myBufferEndOffset;
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
while (pos < lBufferEnd) {
final char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
if (c == '\r' || c == '\n') break;
pos++;
}
return pos;
}
private int getIdentifier(int offset) {
final CharSequence lBuffer = myBuffer;
final char[] lBufferArray = myBufferArray;
int hashCode = (lBufferArray != null ? lBufferArray[offset - 1]:lBuffer.charAt(offset - 1)) * 2;
final int lBufferEnd = myBufferEndOffset;
int pos = offset;
if (pos < lBufferEnd) {
char c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
while (c >= 'a' && c <= 'z' ||
c >= 'A' && c <= 'Z' ||
c >= '0' && c <= '9' ||
c == '_' ||
c == '$' ||
c > 127 && Character.isJavaIdentifierPart(c)) {
pos++;
hashCode += c;
if (pos == lBufferEnd) break;
c = lBufferArray != null ? lBufferArray[pos]:lBuffer.charAt(pos);
}
}
if (myTable.contains(hashCode, lBufferArray, lBuffer, offset - 1)) {
myTokenType = myTable.getTokenType(hashCode);
}
else {
myTokenType = JavaTokenType.IDENTIFIER;
}
return pos;
}
@Override
public CharSequence getBufferSequence() {
return myBuffer;
}
@Override
public final int getBufferEnd() {
return myBufferEndOffset;
}
/*
public static void main(String[] args) throws IOException {
File root = new File(args[0]);
Stats stats = new Stats();
walk(root, stats);
System.out.println("Scanned " + stats.files + " files, total of " + stats.lines + " lines in " + (stats.time / 1000000) + " ms.");
System.out.println("Size:" + stats.bytes);
}
private static void lex(File root, Stats stats) throws IOException {
stats.files++;
BufferedReader reader = new BufferedReader(new FileReader(root));
String s;
StringBuilder buf = new StringBuilder();
while ((s = reader.readLine()) != null) {
stats.lines++;
buf.append(s).append("\n");
}
stats.bytes += buf.length();
long start = System.nanoTime();
lexText(buf);
stats.time += System.nanoTime() - start;
}
private static void lexText(StringBuilder buf) {
JavaLexer lexer = new JavaLexer(LanguageLevel.JDK_1_5);
lexer.start(buf);
while (lexer.getTokenType() != null) {
lexer.advance();
}
}
private static class Stats {
public int files;
public int lines;
public long time;
public long bytes;
}
private static void walk(File root, Stats stats) throws IOException {
if (root.isDirectory()) {
System.out.println("Lexing in " + root.getPath());
for (File file : root.listFiles()) {
walk(file, stats);
}
}
else {
if (root.getName().endsWith(".java")) {
lex(root, stats);
}
}
}
*/
}
java/java-psi-impl/src/com/intellij/psi/impl/PsiElementFactoryImpl.java
+1 -1
View File
@@ -16,9 +16,9 @@
package com.intellij.psi.impl;
import com.intellij.lang.*;
import com.intellij.lang.java.lexer.JavaLexer;
import com.intellij.lang.java.parser.JavaParser;
import com.intellij.lang.java.parser.JavaParserUtil;
import com.intellij.lexer.JavaLexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.pom.java.LanguageLevel;
java/java-psi-impl/src/com/intellij/psi/impl/PsiNameHelperImpl.java
+15 -15
View File
@@ -1,21 +1,21 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.psi.impl;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.lexer.JavaLexer;
import com.intellij.openapi.roots.LanguageLevelProjectExtension;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.pom.java.LanguageLevel;
java/java-psi-impl/src/com/intellij/psi/impl/compiled/ClsParsingUtil.java
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,9 +16,9 @@
package com.intellij.psi.impl.compiled;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.java.lexer.JavaLexer;
import com.intellij.lang.java.parser.JavaParser;
import com.intellij.lang.java.parser.JavaParserUtil;
import com.intellij.lexer.JavaLexer;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.pom.java.LanguageLevel;
java/java-psi-impl/src/com/intellij/psi/impl/light/LightKeyword.java
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2011 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
package com.intellij.psi.impl.light;
import com.intellij.lang.java.JavaLanguage;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.*;
@@ -38,7 +38,7 @@ public class LightKeyword extends LightElement implements PsiKeyword, PsiJavaTok
@Override
public IElementType getTokenType(){
Lexer lexer = new JavaLexer(LanguageLevel.HIGHEST);
Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
lexer.start(myText);
return lexer.getTokenType();
}
java/java-psi-impl/src/com/intellij/psi/impl/source/PsiJavaFileImpl.java
+5 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2011 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
package com.intellij.psi.impl.source;
import com.intellij.ide.highlighter.JavaFileType;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.components.ServiceManager;
import com.intellij.openapi.fileTypes.FileType;
@@ -33,12 +33,13 @@ public class PsiJavaFileImpl extends PsiJavaFileBaseImpl {
super(JavaStubElementTypes.JAVA_FILE, JavaStubElementTypes.JAVA_FILE, file);
}
public String toString(){
@Override
public String toString() {
return "PsiJavaFile:" + getName();
}
public Lexer createLexer() {
return new JavaLexer(getLanguageLevel());
return JavaParserDefinition.createLexer(getLanguageLevel());
}
@NotNull
java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaDocElementType.java
+5 -8
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,17 +19,14 @@ import com.intellij.lang.ASTNode;
import com.intellij.lang.Language;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.java.JavaLanguage;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lang.java.parser.JavaParserUtil;
import com.intellij.lang.java.parser.JavadocParser;
import com.intellij.lexer.JavaLexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.roots.LanguageLevelProjectExtension;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.impl.source.javadoc.PsiDocCommentImpl;
import com.intellij.psi.impl.source.javadoc.PsiDocMethodOrFieldRef;
import com.intellij.psi.impl.source.javadoc.PsiDocParamRef;
import com.intellij.psi.impl.source.javadoc.PsiDocTagImpl;
import com.intellij.psi.impl.source.javadoc.PsiInlineDocTagImpl;
import com.intellij.psi.impl.source.javadoc.*;
import com.intellij.psi.tree.*;
import com.intellij.psi.tree.java.IJavaDocElementType;
import com.intellij.util.ReflectionUtil;
@@ -123,7 +120,7 @@ public interface JavaDocElementType {
@Override
public boolean isParsable(final CharSequence buffer, Language fileLanguage, final Project project) {
final JavaLexer lexer = new JavaLexer(LanguageLevelProjectExtension.getInstance(project).getLanguageLevel());
Lexer lexer = JavaParserDefinition.createLexer(LanguageLevelProjectExtension.getInstance(project).getLanguageLevel());
lexer.start(buffer);
if (lexer.getTokenType() == DOC_COMMENT) {
lexer.advance();
java/java-psi-impl/src/com/intellij/psi/impl/source/tree/JavaElementType.java
+5 -3
View File
@@ -17,8 +17,10 @@ package com.intellij.psi.impl.source.tree;
import com.intellij.lang.*;
import com.intellij.lang.java.JavaLanguage;
import com.intellij.lang.java.parser.*;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lang.java.parser.JavaParser;
import com.intellij.lang.java.parser.JavaParserUtil;
import com.intellij.lang.java.parser.ReferenceParser;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.pom.java.LanguageLevel;
@@ -167,7 +169,7 @@ public interface JavaElementType {
@Override
public int getErrorsCount(final CharSequence seq, Language fileLanguage, final Project project) {
final Lexer lexer = new JavaLexer(LanguageLevel.HIGHEST);
Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
lexer.start(seq);
if (lexer.getTokenType() != JavaTokenType.LBRACE) return IErrorCounterReparseableElementType.FATAL_ERROR;
java/java-tests/testSrc/com/intellij/lang/java/lexer/JavaLexerTest.java
+2 -2
View File
@@ -15,7 +15,7 @@
*/
package com.intellij.lang.java.lexer;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.testFramework.LexerTestCase;
@@ -158,7 +158,7 @@ public class JavaLexerTest extends LexerTestCase {
@Override
protected Lexer createLexer() {
return new JavaLexer(LanguageLevel.HIGHEST);
return JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
}
@Override
java/java-tests/testSrc/com/intellij/lang/java/parser/JavadocParsingTest.java
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2011 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -81,5 +81,5 @@ public class JavadocParsingTest extends JavaParsingTestCase {
public void testTypeParam() { doTest(true); }
public void testParameterlessTag() { doTest(true); }
public void testIDEADEV_41403() {doTest(true);}
public void testIDEADEV_41403() { doTest(true); }
}
java/java-tests/testSrc/com/intellij/lang/psi/PsiBuilderTest.java
+20 -7
View File
@@ -1,8 +1,23 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lang.psi;
import com.intellij.lang.*;
import com.intellij.lang.impl.PsiBuilderImpl;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.openapi.fileTypes.StdFileTypes;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
@@ -17,7 +32,7 @@ import com.intellij.testFramework.LightIdeaTestCase;
import junit.framework.AssertionFailedError;
/**
* Date: Jan 21, 2005
* @since Jan 21, 2005
* @author max
*/
public class PsiBuilderTest extends LightIdeaTestCase {
@@ -66,7 +81,7 @@ public class PsiBuilderTest extends LightIdeaTestCase {
final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
assertNotNull(parserDefinition);
PsiFile psiFile = createFile("x.java", text);
return new PsiBuilderImpl(getProject(), psiFile, parserDefinition, new JavaLexer(LanguageLevel.JDK_1_5),
return new PsiBuilderImpl(getProject(), psiFile, parserDefinition, JavaParserDefinition.createLexer(LanguageLevel.JDK_1_5),
SharedImplUtil.findCharTableByTree(psiFile.getNode()), text, originalTree, null);
}
@@ -152,8 +167,8 @@ public class PsiBuilderTest extends LightIdeaTestCase {
public void testAssertionFailureOnUnbalancedMarkers() {
myBuilder = createBuilder("foo");
myBuilder.setDebugMode(true);
final PsiBuilder.Marker m = myBuilder.mark();
final PsiBuilder.Marker m1 = myBuilder.mark();
PsiBuilder.Marker m = myBuilder.mark();
@SuppressWarnings("UnusedDeclaration") PsiBuilder.Marker m1 = myBuilder.mark();
myBuilder.getTokenType();
myBuilder.advanceLexer();
try {
@@ -227,6 +242,4 @@ public class PsiBuilderTest extends LightIdeaTestCase {
myBuilder.advanceLexer();
root.done(JavaStubElementTypes.ENUM_CONSTANT_INITIALIZER);
}
}
java/java-tests/testSrc/com/intellij/lexer/LayeredLexerTest.java
+19 -4
View File
@@ -1,5 +1,21 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.tree.IElementType;
@@ -91,17 +107,16 @@ public class LayeredLexerTest extends TestCase {
assertEquals(null, lexer.getTokenType());
}
private Lexer setupLexer(String text) {
LayeredLexer lexer = new LayeredLexer(new JavaLexer(LanguageLevel.JDK_1_3));
private static Lexer setupLexer(String text) {
LayeredLexer lexer = new LayeredLexer(JavaParserDefinition.createLexer(LanguageLevel.JDK_1_3));
lexer.registerSelfStoppingLayer(new StringLiteralLexer('\"', JavaTokenType.STRING_LITERAL),
new IElementType[]{JavaTokenType.STRING_LITERAL},
IElementType.EMPTY_ARRAY);
lexer.start(text);
return lexer;
}
private String nextToken(Lexer lexer) {
private static String nextToken(Lexer lexer) {
assertTrue(lexer.getTokenType() != null);
final String s = lexer.getBufferSequence().subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
lexer.advance();
platform/core-api/src/com/intellij/lexer/DocCommentLexer.java
+2 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import com.intellij.util.text.CharArrayUtil;
import java.io.IOException;
/** @deprecated use com.intellij.lang.java.JavaDocLexer (to remove in IDEA 14) */
public class DocCommentLexer extends MergingLexerAdapter {
public DocCommentLexer(final DocCommentTokenTypes tokenTypes, final boolean isJdk15Enabled) {
super(new AsteriskStripperLexer(new _JavaDocLexer(isJdk15Enabled, tokenTypes), tokenTypes),
platform/core-api/src/com/intellij/lexer/_JavaDocLexer.java
+3 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,8 +21,9 @@ package com.intellij.lexer;
import com.intellij.psi.tree.IElementType;
/**
* @deprecated use com.intellij.lang.java.JavaDocLexer (to remove in IDEA 14)
*
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.1
* on 15.09.08 21:08 from the specification file
plugins/ui-designer/src/com/intellij/uiDesigner/make/FormSourceCodeGenerator.java
+7 -6
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,8 @@
*/
package com.intellij.uiDesigner.make;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.application.ApplicationNamesInfo;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.editor.Document;
@@ -374,10 +375,10 @@ public final class FormSourceCodeGenerator {
}
private static boolean lexemsEqual(final PsiClass classToBind, final PsiClass newClass) {
JavaLexer oldTextLexer = new JavaLexer(LanguageLevel.HIGHEST);
JavaLexer newTextLexer = new JavaLexer(LanguageLevel.HIGHEST);
final String oldBuffer = classToBind.getText();
final String newBuffer = newClass.getText();
Lexer oldTextLexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
Lexer newTextLexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
String oldBuffer = classToBind.getText();
String newBuffer = newClass.getText();
oldTextLexer.start(oldBuffer);
newTextLexer.start(newBuffer);