diff --git a/java/java-impl/src/com/intellij/lang/java/parser/DeclarationParser.java b/java/java-impl/src/com/intellij/lang/java/parser/DeclarationParser.java index 0bd4cf719fd7..35c5d379cefd 100644 --- a/java/java-impl/src/com/intellij/lang/java/parser/DeclarationParser.java +++ b/java/java-impl/src/com/intellij/lang/java/parser/DeclarationParser.java @@ -116,7 +116,7 @@ public class DeclarationParser { } } - declaration.done(JavaElementType.CLASS); + done(declaration, JavaElementType.CLASS); return declaration; } @@ -162,10 +162,10 @@ public class DeclarationParser { if (builder.getTokenType() == JavaTokenType.LBRACE) { final PsiBuilder.Marker constantInit = builder.mark(); parseClassBodyWithBraces(builder, false, false); - constantInit.done(JavaElementType.ENUM_CONSTANT_INITIALIZER); + done(constantInit, JavaElementType.ENUM_CONSTANT_INITIALIZER); } - constant.done(JavaElementType.ENUM_CONSTANT); + done(constant, JavaElementType.ENUM_CONSTANT); return constant; } else { @@ -309,7 +309,7 @@ public class DeclarationParser { final PsiBuilder.Marker error = typeParams.precede(); error.errorBefore(JavaErrorMessages.message("unexpected.token"), codeBlock); } - declaration.done(JavaElementType.CLASS_INITIALIZER); + done(declaration, JavaElementType.CLASS_INITIALIZER); return declaration; } else { @@ -391,7 +391,7 @@ public class DeclarationParser { } } - modList.done(JavaElementType.MODIFIER_LIST); + done(modList, JavaElementType.MODIFIER_LIST); return Pair.create(modList, isEmpty); } @@ -405,7 +405,7 @@ public class DeclarationParser { final PsiBuilder.Marker receiver = builder.mark(); final PsiBuilder.Marker annotations = parseAnnotations(builder); if (annotations != null) { - receiver.done(JavaElementType.METHOD_RECEIVER); + done(receiver, JavaElementType.METHOD_RECEIVER); } else { receiver.drop(); @@ -441,7 +441,7 @@ public class DeclarationParser { } } - declaration.done(anno ? JavaElementType.ANNOTATION_METHOD : JavaElementType.METHOD); + done(declaration, anno ? JavaElementType.ANNOTATION_METHOD : JavaElementType.METHOD); return declaration; } @@ -528,7 +528,7 @@ public class DeclarationParser { invalidElements.error(errorMessage); } - paramList.done(JavaElementType.PARAMETER_LIST); + done(paramList, JavaElementType.PARAMETER_LIST); return paramList; } @@ -552,7 +552,7 @@ public class DeclarationParser { if (expect(builder, JavaTokenType.IDENTIFIER)) { eatBrackets(builder); - param.done(JavaElementType.PARAMETER); + done(param, JavaElementType.PARAMETER); return param; } else { @@ -603,7 +603,7 @@ public class DeclarationParser { } if (builder.getTokenType() != JavaTokenType.COMMA) break; - variable.done(varType); + done(variable, varType); builder.advanceLexer(); if (builder.getTokenType() != JavaTokenType.IDENTIFIER) { @@ -642,7 +642,7 @@ public class DeclarationParser { } if (openMarker) { - variable.done(varType); + done(variable, varType); } return declaration; @@ -683,7 +683,7 @@ public class DeclarationParser { parseAnnotationParameterList(builder); - anno.done(JavaElementType.ANNOTATION); + done(anno, JavaElementType.ANNOTATION); return anno; } @@ -692,12 +692,12 @@ public class DeclarationParser { PsiBuilder.Marker list = builder.mark(); if (!expect(builder, JavaTokenType.LPARENTH)) { - list.done(JavaElementType.ANNOTATION_PARAMETER_LIST); + done(list, JavaElementType.ANNOTATION_PARAMETER_LIST); return list; } if (expect(builder, JavaTokenType.RPARENTH)) { - list.done(JavaElementType.ANNOTATION_PARAMETER_LIST); + done(list, JavaElementType.ANNOTATION_PARAMETER_LIST); return list; } @@ -739,7 +739,7 @@ public class DeclarationParser { } } - list.done(JavaElementType.ANNOTATION_PARAMETER_LIST); + done(list, JavaElementType.ANNOTATION_PARAMETER_LIST); return list; } @@ -749,7 +749,7 @@ public class DeclarationParser { if (mayBeSimple) { parseAnnotationValue(builder); if (builder.getTokenType() != JavaTokenType.EQ) { - pair.done(JavaElementType.NAME_VALUE_PAIR); + done(pair, JavaElementType.NAME_VALUE_PAIR); return false; } @@ -763,7 +763,7 @@ public class DeclarationParser { parseAnnotationValue(builder); - pair.done(JavaElementType.NAME_VALUE_PAIR); + done(pair, JavaElementType.NAME_VALUE_PAIR); return hasName; } @@ -798,7 +798,7 @@ public class DeclarationParser { builder.advanceLexer(); if (expect(builder, JavaTokenType.RBRACE)) { - annoArray.done(JavaElementType.ANNOTATION_ARRAY_INITIALIZER); + done(annoArray, JavaElementType.ANNOTATION_ARRAY_INITIALIZER); return annoArray; } @@ -819,7 +819,7 @@ public class DeclarationParser { } } - annoArray.done(JavaElementType.ANNOTATION_ARRAY_INITIALIZER); + done(annoArray, JavaElementType.ANNOTATION_ARRAY_INITIALIZER); if (unclosed) { annoArray.setCustomEdgeProcessors(null, GREEDY_RIGHT_EDGE_PROCESSOR); } diff --git a/java/java-impl/src/com/intellij/lang/java/parser/FileParser.java b/java/java-impl/src/com/intellij/lang/java/parser/FileParser.java index f55fde673840..329ae1f7ed09 100644 --- a/java/java-impl/src/com/intellij/lang/java/parser/FileParser.java +++ b/java/java-impl/src/com/intellij/lang/java/parser/FileParser.java @@ -27,38 +27,61 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import static com.intellij.lang.PsiBuilderUtil.expect; +import static com.intellij.lang.java.parser.JavaParserUtil.done; import static com.intellij.lang.java.parser.JavaParserUtil.semicolon; public class FileParser { + public interface TopLevelDeclarationParser { + @Nullable PsiBuilder.Marker parse(PsiBuilder builder); + } + private static final TokenSet IMPORT_LIST_STOPPER_SET = TokenSet.orSet( ElementType.MODIFIER_BIT_SET, TokenSet.create(JavaTokenType.CLASS_KEYWORD, JavaTokenType.INTERFACE_KEYWORD, JavaTokenType.ENUM_KEYWORD, JavaTokenType.AT)); + private static final TopLevelDeclarationParser TOP_LEVEL_DECLARATION_PARSER = new TopLevelDeclarationParser() { + public PsiBuilder.Marker parse(final PsiBuilder builder) { + return DeclarationParser.parse(builder, DeclarationParser.Context.FILE); + } + }; + private FileParser() { } public static void parse(final PsiBuilder builder) { + parseFile(builder, IMPORT_LIST_STOPPER_SET, TOP_LEVEL_DECLARATION_PARSER, JavaErrorMessages.message("expected.class.or.interface")); + } + + public static void parseFile(final PsiBuilder builder, final TokenSet importListStoppers, + final TopLevelDeclarationParser declarationParser, final String errorMessage) { parsePackageStatement(builder); - parseImportList(builder); + final Pair impListInfo = parseImportList(builder, importListStoppers); + Boolean firstDeclarationOk = null; + PsiBuilder.Marker firstDeclaration = null; PsiBuilder.Marker invalidElements = null; while (!builder.eof()) { if (builder.getTokenType() == JavaTokenType.SEMICOLON) { if (invalidElements != null) { - invalidElements.error(JavaErrorMessages.message("expected.class.or.interface")); + invalidElements.error(errorMessage); invalidElements = null; } builder.advanceLexer(); + if (firstDeclarationOk == null) firstDeclarationOk = false; continue; } - final PsiBuilder.Marker declaration = DeclarationParser.parse(builder, DeclarationParser.Context.FILE); + final PsiBuilder.Marker declaration = declarationParser.parse(builder); if (declaration != null) { if (invalidElements != null) { - invalidElements.errorBefore(JavaErrorMessages.message("expected.class.or.interface"), declaration); + invalidElements.errorBefore(errorMessage, declaration); invalidElements = null; } + if (firstDeclarationOk == null) { + firstDeclarationOk = true; + firstDeclaration = declaration; + } continue; } @@ -66,10 +89,16 @@ public class FileParser { invalidElements = builder.mark(); } builder.advanceLexer(); + if (firstDeclarationOk == null) firstDeclarationOk = false; } if (invalidElements != null) { - invalidElements.error(JavaErrorMessages.message("expected.class.or.interface")); + invalidElements.error(errorMessage); + } + + if (impListInfo.second && firstDeclarationOk == Boolean.TRUE) { + impListInfo.first.setCustomEdgeProcessors(JavaParserUtil.PRECEDING_COMMENT_BINDER, null); // pass comments behind fake import list + firstDeclaration.setCustomEdgeProcessors(JavaParserUtil.SPECIAL_PRECEDING_COMMENT_BINDER, null); } } @@ -80,7 +109,7 @@ public class FileParser { if (!expect(builder, JavaTokenType.PACKAGE_KEYWORD)) { final PsiBuilder.Marker modList = builder.mark(); DeclarationParser.parseAnnotations(builder); - modList.done(JavaElementType.MODIFIER_LIST); + done(modList, JavaElementType.MODIFIER_LIST); if (!expect(builder, JavaTokenType.PACKAGE_KEYWORD)) { statement.rollbackTo(); return null; @@ -95,15 +124,10 @@ public class FileParser { semicolon(builder); - statement.done(JavaElementType.PACKAGE_STATEMENT); + done(statement, JavaElementType.PACKAGE_STATEMENT); return statement; } - @NotNull - private static Pair parseImportList(final PsiBuilder builder) { - return parseImportList(builder, IMPORT_LIST_STOPPER_SET); - } - @NotNull public static Pair parseImportList(final PsiBuilder builder, final TokenSet stoppers) { final PsiBuilder.Marker list = builder.mark(); @@ -134,7 +158,7 @@ public class FileParser { } } - list.done(JavaElementType.IMPORT_LIST); + done(list, JavaElementType.IMPORT_LIST); return Pair.create(list, isEmpty); } @@ -153,7 +177,7 @@ public class FileParser { semicolon(builder); } - statement.done(type); + done(statement, type); return statement; } } diff --git a/java/java-impl/src/com/intellij/lang/java/parser/JavaParser.java b/java/java-impl/src/com/intellij/lang/java/parser/JavaParser.java index 3bd61235b27d..dbe1ab0f5121 100644 --- a/java/java-impl/src/com/intellij/lang/java/parser/JavaParser.java +++ b/java/java-impl/src/com/intellij/lang/java/parser/JavaParser.java @@ -19,7 +19,6 @@ import com.intellij.lang.ASTNode; import com.intellij.lang.PsiBuilder; import com.intellij.lang.PsiParser; import com.intellij.pom.java.LanguageLevel; -import com.intellij.psi.impl.source.parsing.ParseUtil; import com.intellij.psi.tree.IElementType; import org.jetbrains.annotations.NotNull; @@ -41,8 +40,6 @@ public class JavaParser implements PsiParser { FileParser.parse(builder); root.done(rootType); - final ASTNode rootNode = builder.getTreeBuilt(); - ParseUtil.bindComments(rootNode); - return rootNode; + return builder.getTreeBuilt(); } } diff --git a/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java b/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java index e4120a033c54..d4c66df0241a 100644 --- a/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java +++ b/java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java @@ -18,8 +18,12 @@ package com.intellij.lang.java.parser; import com.intellij.codeInsight.daemon.JavaErrorMessages; import com.intellij.lang.*; import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.text.StringUtil; import com.intellij.pom.java.LanguageLevel; import com.intellij.psi.JavaTokenType; +import com.intellij.psi.impl.source.tree.ElementType; +import com.intellij.psi.impl.source.tree.JavaDocElementType; +import com.intellij.psi.impl.source.tree.JavaElementType; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; import com.intellij.util.diff.FlyweightCapableTreeStructure; @@ -34,11 +38,77 @@ public class JavaParserUtil { private static final Key LANG_LEVEL_KEY = Key.create("JavaParserUtil.LanguageLevel"); public static final WhitespacesAndCommentsProcessor GREEDY_RIGHT_EDGE_PROCESSOR = new WhitespacesAndCommentsProcessor() { - public int process(final List tokens) { + public int process(final List tokens, final boolean atStreamEdge, final TokenTextGetter getter) { return tokens.size(); } }; + private static class PrecedingWhitespacesAndCommentsProcessor implements WhitespacesAndCommentsProcessor { + private final boolean myAfterEmptyImport; + + public PrecedingWhitespacesAndCommentsProcessor(final boolean afterImport) { + this.myAfterEmptyImport = afterImport; + } + + public int process(final List tokens, final boolean atStreamEdge, final TokenTextGetter + getter) { + if (tokens.size() == 0) return 0; + + // 1. bind doc comment + for (int idx = tokens.size() - 1; idx >= 0; idx--) { + if (tokens.get(idx) == JavaDocElementType.DOC_COMMENT) return idx; + } + + // 2. bind plain comments + int result = tokens.size(); + for (int idx = tokens.size() - 1; idx >= 0; idx--) { + final IElementType tokenType = tokens.get(idx); + if (ElementType.JAVA_WHITESPACE_BIT_SET.contains(tokenType)) { + if (StringUtil.getLineBreakCount(getter.get(idx)) > 1) break; + } + else if (ElementType.JAVA_PLAIN_COMMENT_BIT_SET.contains(tokenType)) { + if (atStreamEdge || + (idx == 0 && myAfterEmptyImport) || + (idx > 0 && ElementType.JAVA_WHITESPACE_BIT_SET.contains(tokens.get(idx - 1)) && StringUtil.containsLineBreak(getter.get(idx - 1)))) { + result = idx; + } + } + else break; + } + + return result; + } + } + + private static class TrailingWhitespacesAndCommentsProcessor implements WhitespacesAndCommentsProcessor { + public int process(final List tokens, final boolean atStreamEdge, final TokenTextGetter getter) { + if (tokens.size() == 0) return 0; + + int result = 0; + for (int idx = 0; idx < tokens.size(); idx++) { + final IElementType tokenType = tokens.get(idx); + if (ElementType.JAVA_WHITESPACE_BIT_SET.contains(tokenType)) { + if (StringUtil.containsLineBreak(getter.get(idx))) break; + } + else if (ElementType.JAVA_PLAIN_COMMENT_BIT_SET.contains(tokenType)) { + result = idx + 1; + } + else break; + } + + return result; + } + } + + private static final TokenSet PRECEDING_COMMENT_SET = ElementType.FULL_MEMBER_BIT_SET; + private static final TokenSet TRAILING_COMMENT_SET = TokenSet.orSet( + TokenSet.create(JavaElementType.PACKAGE_STATEMENT), + ElementType.IMPORT_STATEMENT_BASE_BIT_SET, ElementType.FULL_MEMBER_BIT_SET, ElementType.JAVA_STATEMENT_BIT_SET); + + public static final WhitespacesAndCommentsProcessor PRECEDING_COMMENT_BINDER = new PrecedingWhitespacesAndCommentsProcessor(false); + public static final WhitespacesAndCommentsProcessor SPECIAL_PRECEDING_COMMENT_BINDER = new PrecedingWhitespacesAndCommentsProcessor(true); + public static final WhitespacesAndCommentsProcessor TRAILING_COMMENT_BINDER = new TrailingWhitespacesAndCommentsProcessor(); + private JavaParserUtil() { } public static void setLanguageLevel(final PsiBuilder builder, final LanguageLevel level) { @@ -57,12 +127,19 @@ public class JavaParserUtil { return level; } + public static void done(final PsiBuilder.Marker marker, final IElementType type) { + marker.done(type); + final WhitespacesAndCommentsProcessor left = PRECEDING_COMMENT_SET.contains(type) ? PRECEDING_COMMENT_BINDER : null; + final WhitespacesAndCommentsProcessor right = TRAILING_COMMENT_SET.contains(type) ? TRAILING_COMMENT_BINDER : null; + marker.setCustomEdgeProcessors(left, right); + } + @Nullable public static IElementType exprType(@Nullable final PsiBuilder.Marker marker) { return marker != null ? ((LighterASTNode)marker).getTokenType() : null; } - // used instead of PsiBuilder.error() as it drops all but first subsequent error messages + // used instead of PsiBuilder.error() as it keeps all subsequent error messages public static void error(final PsiBuilder builder, final String message) { builder.mark().error(message); } diff --git a/java/java-impl/src/com/intellij/lang/java/parser/StatementParser.java b/java/java-impl/src/com/intellij/lang/java/parser/StatementParser.java index 7d592df0e92e..b0cc604745a8 100644 --- a/java/java-impl/src/com/intellij/lang/java/parser/StatementParser.java +++ b/java/java-impl/src/com/intellij/lang/java/parser/StatementParser.java @@ -118,7 +118,7 @@ public class StatementParser { final boolean greedyBlock = !expectOrError(builder, JavaTokenType.RBRACE, JavaErrorMessages.message("expected.rbrace")); - codeBlock.done(JavaElementType.CODE_BLOCK); + done(codeBlock, JavaElementType.CODE_BLOCK); if (greedyBlock) { codeBlock.setCustomEdgeProcessors(null, GREEDY_RIGHT_EDGE_PROCESSOR); } @@ -214,7 +214,7 @@ public class StatementParser { else if (tokenType == JavaTokenType.SEMICOLON) { final PsiBuilder.Marker empty = builder.mark(); builder.advanceLexer(); - empty.done(JavaElementType.EMPTY_STATEMENT); + done(empty, JavaElementType.EMPTY_STATEMENT); return empty; } else if (tokenType == JavaTokenType.IDENTIFIER || tokenType == JavaTokenType.AT) { @@ -230,7 +230,7 @@ public class StatementParser { ReferenceParser.parseType(builder, false, false); error(builder, JavaErrorMessages.message("expected.identifier")); } - declStatement.done(JavaElementType.DECLARATION_STATEMENT); + done(declStatement, JavaElementType.DECLARATION_STATEMENT); return declStatement; } } @@ -255,15 +255,15 @@ public class StatementParser { } if (count > 1) { pos.drop(); - list.done(JavaElementType.EXPRESSION_LIST); + done(list, JavaElementType.EXPRESSION_LIST); semicolon(builder); - statement.done(JavaElementType.EXPRESSION_LIST_STATEMENT); + done(statement, JavaElementType.EXPRESSION_LIST_STATEMENT); return statement; } if (exprType(expr) != JavaElementType.REFERENCE_EXPRESSION) { drop(list, pos); semicolon(builder); - statement.done(JavaElementType.EXPRESSION_STATEMENT); + done(statement, JavaElementType.EXPRESSION_STATEMENT); return statement; } pos.rollbackTo(); @@ -275,7 +275,7 @@ public class StatementParser { final PsiBuilder.Marker decl = DeclarationParser.parse(builder, DeclarationParser.Context.CODE_BLOCK); if (decl != null) { final PsiBuilder.Marker statement = decl.precede(); - statement.done(JavaElementType.DECLARATION_STATEMENT); + done(statement, JavaElementType.DECLARATION_STATEMENT); return statement; } @@ -283,7 +283,7 @@ public class StatementParser { final PsiBuilder.Marker statement = builder.mark(); advance(builder, 2); parseStatement(builder); - statement.done(JavaElementType.LABELED_STATEMENT); + done(statement, JavaElementType.LABELED_STATEMENT); return statement; } @@ -291,7 +291,7 @@ public class StatementParser { final PsiBuilder.Marker statement = builder.mark(); ExpressionParser.parse(builder); semicolon(builder); - statement.done(JavaElementType.EXPRESSION_STATEMENT); + done(statement, JavaElementType.EXPRESSION_STATEMENT); return statement; } @@ -311,19 +311,19 @@ public class StatementParser { builder.advanceLexer(); if (!parseExpressionInParenth(builder)) { - statement.done(JavaElementType.IF_STATEMENT); + done(statement, JavaElementType.IF_STATEMENT); return statement; } final PsiBuilder.Marker thenStatement = parseStatement(builder); if (thenStatement == null) { error(builder, JavaErrorMessages.message("expected.statement")); - statement.done(JavaElementType.IF_STATEMENT); + done(statement, JavaElementType.IF_STATEMENT); return statement; } if (!expect(builder, JavaTokenType.ELSE_KEYWORD)) { - statement.done(JavaElementType.IF_STATEMENT); + done(statement, JavaElementType.IF_STATEMENT); return statement; } @@ -332,7 +332,7 @@ public class StatementParser { error(builder, JavaErrorMessages.message("expected.statement")); } - statement.done(JavaElementType.IF_STATEMENT); + done(statement, JavaElementType.IF_STATEMENT); return statement; } @@ -342,7 +342,7 @@ public class StatementParser { builder.advanceLexer(); if (!parseExpressionInParenth(builder)) { - statement.done(JavaElementType.WHILE_STATEMENT); + done(statement, JavaElementType.WHILE_STATEMENT); return statement; } @@ -351,7 +351,7 @@ public class StatementParser { error(builder, JavaErrorMessages.message("expected.statement")); } - statement.done(JavaElementType.WHILE_STATEMENT); + done(statement, JavaElementType.WHILE_STATEMENT); return statement; } @@ -362,7 +362,7 @@ public class StatementParser { if (!expect(builder, JavaTokenType.LPARENTH)) { error(builder, JavaErrorMessages.message("expected.lparen")); - statement.done(JavaElementType.FOR_STATEMENT); + done(statement, JavaElementType.FOR_STATEMENT); return statement; } @@ -384,7 +384,7 @@ public class StatementParser { if (init == null){ error(builder, JavaErrorMessages.message("expected.statement")); if (!expect(builder, JavaTokenType.RPARENTH)) { - statement.done(JavaElementType.FOR_STATEMENT); + done(statement, JavaElementType.FOR_STATEMENT); return statement; } } @@ -393,7 +393,7 @@ public class StatementParser { if (!expect(builder, JavaTokenType.SEMICOLON)) { error(builder, JavaErrorMessages.message("expected.semicolon")); if (!expect(builder, JavaTokenType.RPARENTH)) { - statement.done(JavaElementType.FOR_STATEMENT); + done(statement, JavaElementType.FOR_STATEMENT); return statement; } } @@ -401,7 +401,7 @@ public class StatementParser { parseExpressionOrExpressionList(builder); if (!expect(builder, JavaTokenType.RPARENTH)) { error(builder, JavaErrorMessages.message("expected.rparen")); - statement.done(JavaElementType.FOR_STATEMENT); + done(statement, JavaElementType.FOR_STATEMENT); return statement; } } @@ -412,7 +412,7 @@ public class StatementParser { error(builder, JavaErrorMessages.message("expected.statement")); } - statement.done(JavaElementType.FOR_STATEMENT); + done(statement, JavaElementType.FOR_STATEMENT); return statement; } @@ -423,7 +423,7 @@ public class StatementParser { final PsiBuilder.Marker expressionStatement; if (builder.getTokenType() != JavaTokenType.COMMA) { expressionStatement = expr.precede(); - expressionStatement.done(JavaElementType.EXPRESSION_STATEMENT); + done(expressionStatement, JavaElementType.EXPRESSION_STATEMENT); } else { final PsiBuilder.Marker expressionList = expr.precede(); @@ -438,8 +438,8 @@ public class StatementParser { } while (builder.getTokenType() == JavaTokenType.COMMA); - expressionList.done(JavaElementType.EXPRESSION_LIST); - expressionStatement.done(JavaElementType.EXPRESSION_LIST_STATEMENT); + done(expressionList, JavaElementType.EXPRESSION_LIST); + done(expressionStatement, JavaElementType.EXPRESSION_LIST_STATEMENT); } } @@ -462,7 +462,7 @@ public class StatementParser { error(builder, JavaErrorMessages.message("expected.rparen")); } - statement.done(JavaElementType.FOREACH_STATEMENT); + done(statement, JavaElementType.FOREACH_STATEMENT); return statement; } @@ -474,13 +474,13 @@ public class StatementParser { final PsiBuilder.Marker body = parseStatement(builder); if (body == null) { error(builder, JavaErrorMessages.message("expected.statement")); - statement.done(JavaElementType.DO_WHILE_STATEMENT); + done(statement, JavaElementType.DO_WHILE_STATEMENT); return statement; } if (!expect(builder, JavaTokenType.WHILE_KEYWORD)) { error(builder, JavaErrorMessages.message("expected.while")); - statement.done(JavaElementType.DO_WHILE_STATEMENT); + done(statement, JavaElementType.DO_WHILE_STATEMENT); return statement; } @@ -488,7 +488,7 @@ public class StatementParser { semicolon(builder); } - statement.done(JavaElementType.DO_WHILE_STATEMENT); + done(statement, JavaElementType.DO_WHILE_STATEMENT); return statement; } @@ -498,7 +498,7 @@ public class StatementParser { builder.advanceLexer(); if (!parseExpressionInParenth(builder)) { - statement.done(JavaElementType.SWITCH_STATEMENT); + done(statement, JavaElementType.SWITCH_STATEMENT); return statement; } @@ -507,7 +507,7 @@ public class StatementParser { error(builder, JavaErrorMessages.message("expected.lbrace")); } - statement.done(JavaElementType.SWITCH_STATEMENT); + done(statement, JavaElementType.SWITCH_STATEMENT); return statement; } @@ -527,7 +527,7 @@ public class StatementParser { expectOrError(builder, JavaTokenType.COLON, JavaErrorMessages.message("expected.colon")); - statement.done(JavaElementType.SWITCH_LABEL_STATEMENT); + done(statement, JavaElementType.SWITCH_LABEL_STATEMENT); return statement; } @@ -537,7 +537,7 @@ public class StatementParser { builder.advanceLexer(); expect(builder, JavaTokenType.IDENTIFIER); semicolon(builder); - statement.done(JavaElementType.BREAK_STATEMENT); + done(statement, JavaElementType.BREAK_STATEMENT); return statement; } @@ -547,7 +547,7 @@ public class StatementParser { builder.advanceLexer(); expect(builder, JavaTokenType.IDENTIFIER); semicolon(builder); - statement.done(JavaElementType.CONTINUE_STATEMENT); + done(statement, JavaElementType.CONTINUE_STATEMENT); return statement; } @@ -559,7 +559,7 @@ public class StatementParser { ExpressionParser.parse(builder); semicolon(builder); - statement.done(JavaElementType.RETURN_STATEMENT); + done(statement, JavaElementType.RETURN_STATEMENT); return statement; } @@ -571,12 +571,12 @@ public class StatementParser { final PsiBuilder.Marker expr = ExpressionParser.parse(builder); if (expr == null) { error(builder, JavaErrorMessages.message("expected.expression")); - statement.done(JavaElementType.THROW_STATEMENT); + done(statement, JavaElementType.THROW_STATEMENT); return statement; } semicolon(builder); - statement.done(JavaElementType.THROW_STATEMENT); + done(statement, JavaElementType.THROW_STATEMENT); return statement; } @@ -586,7 +586,7 @@ public class StatementParser { builder.advanceLexer(); if (!parseExpressionInParenth(builder)) { - statement.done(JavaElementType.SYNCHRONIZED_STATEMENT); + done(statement, JavaElementType.SYNCHRONIZED_STATEMENT); return statement; } @@ -595,7 +595,7 @@ public class StatementParser { error(builder, JavaErrorMessages.message("expected.lbrace")); } - statement.done(JavaElementType.SYNCHRONIZED_STATEMENT); + done(statement, JavaElementType.SYNCHRONIZED_STATEMENT); return statement; } @@ -607,13 +607,13 @@ public class StatementParser { final PsiBuilder.Marker tryBlock = parseCodeBlock(builder); if (tryBlock == null) { error(builder, JavaErrorMessages.message("expected.lbrace")); - statement.done(JavaElementType.TRY_STATEMENT); + done(statement, JavaElementType.TRY_STATEMENT); return statement; } if (!TRY_CLOSERS_SET.contains(builder.getTokenType())) { error(builder, JavaErrorMessages.message("expected.catch.or.finally")); - statement.done(JavaElementType.TRY_STATEMENT); + done(statement, JavaElementType.TRY_STATEMENT); return statement; } @@ -628,7 +628,7 @@ public class StatementParser { } } - statement.done(JavaElementType.TRY_STATEMENT); + done(statement, JavaElementType.TRY_STATEMENT); return statement; } @@ -638,7 +638,7 @@ public class StatementParser { if (!expect(builder, JavaTokenType.LPARENTH)) { error(builder, JavaErrorMessages.message("expected.lparen")); - section.done(JavaElementType.CATCH_SECTION); + done(section, JavaElementType.CATCH_SECTION); return false; } @@ -649,18 +649,18 @@ public class StatementParser { if (!expect(builder, JavaTokenType.RPARENTH)) { error(builder, JavaErrorMessages.message("expected.rparen")); - section.done(JavaElementType.CATCH_SECTION); + done(section, JavaElementType.CATCH_SECTION); return false; } final PsiBuilder.Marker body = parseCodeBlock(builder); if (body == null) { error(builder, JavaErrorMessages.message("expected.lbrace")); - section.done(JavaElementType.CATCH_SECTION); + done(section, JavaElementType.CATCH_SECTION); return false; } - section.done(JavaElementType.CATCH_SECTION); + done(section, JavaElementType.CATCH_SECTION); return true; } @@ -672,7 +672,7 @@ public class StatementParser { final PsiBuilder.Marker expr = ExpressionParser.parse(builder); if (expr == null) { error(builder, JavaErrorMessages.message("expected.boolean.expression")); - statement.done(JavaElementType.ASSERT_STATEMENT); + done(statement, JavaElementType.ASSERT_STATEMENT); return statement; } @@ -680,13 +680,13 @@ public class StatementParser { final PsiBuilder.Marker expr2 = ExpressionParser.parse(builder); if (expr2 == null) { error(builder, JavaErrorMessages.message("expected.expression")); - statement.done(JavaElementType.ASSERT_STATEMENT); + done(statement, JavaElementType.ASSERT_STATEMENT); return statement; } } semicolon(builder); - statement.done(JavaElementType.ASSERT_STATEMENT); + done(statement, JavaElementType.ASSERT_STATEMENT); return statement; } @@ -694,7 +694,7 @@ public class StatementParser { private static PsiBuilder.Marker parseBlockStatement(final PsiBuilder builder) { final PsiBuilder.Marker statement = builder.mark(); parseCodeBlock(builder); - statement.done(JavaElementType.BLOCK_STATEMENT); + done(statement, JavaElementType.BLOCK_STATEMENT); return statement; } diff --git a/java/java-impl/src/com/intellij/psi/impl/source/parsing/ParseUtil.java b/java/java-impl/src/com/intellij/psi/impl/source/parsing/ParseUtil.java index 3809e11a8245..72d1a07d4010 100644 --- a/java/java-impl/src/com/intellij/psi/impl/source/parsing/ParseUtil.java +++ b/java/java-impl/src/com/intellij/psi/impl/source/parsing/ParseUtil.java @@ -52,10 +52,6 @@ public class ParseUtil extends ParseUtilBase { inserter.invoke(); } - public static void bindComments(final ASTNode root) { - JavaMissingTokenInserter.bindComments(root); - } - private static class JavaMissingTokenInserter extends MissingTokenInserter { public JavaMissingTokenInserter(final CompositeElement root, final Lexer lexer, diff --git a/java/java-tests/testSrc/com/intellij/lang/java/parser/JavaParsingTestCase.java b/java/java-tests/testSrc/com/intellij/lang/java/parser/JavaParsingTestCase.java index be6205d0a724..06be1084123e 100644 --- a/java/java-tests/testSrc/com/intellij/lang/java/parser/JavaParsingTestCase.java +++ b/java/java-tests/testSrc/com/intellij/lang/java/parser/JavaParsingTestCase.java @@ -23,13 +23,9 @@ import com.intellij.openapi.fileTypes.StdFileTypes; import com.intellij.openapi.project.Project; import com.intellij.openapi.roots.LanguageLevelProjectExtension; import com.intellij.pom.java.LanguageLevel; -import com.intellij.psi.FileViewProvider; -import com.intellij.psi.PsiFile; -import com.intellij.psi.PsiManager; -import com.intellij.psi.SingleRootFileViewProvider; +import com.intellij.psi.*; import com.intellij.psi.impl.JavaPsiFacadeEx; import com.intellij.psi.impl.source.PsiJavaFileImpl; -import com.intellij.psi.impl.source.parsing.ParseUtil; import com.intellij.psi.impl.source.tree.FileElement; import com.intellij.psi.tree.IFileElementType; import com.intellij.testFramework.IdeaTestCase; @@ -97,7 +93,6 @@ public abstract class JavaParsingTestCase extends ParsingTestCase { root.done(this); final ASTNode rootNode = builder.getTreeBuilt(); - ParseUtil.bindComments(rootNode); return rootNode.getFirstChildNode(); } }; @@ -116,7 +111,9 @@ public abstract class JavaParsingTestCase extends ParsingTestCase { } private static PsiBuilder createBuilder(final ASTNode chameleon) { - final Project project = chameleon.getPsi().getProject(); + final PsiElement psi = chameleon.getPsi(); + assert psi != null : chameleon; + final Project project = psi.getProject(); final PsiBuilderFactory factory = PsiBuilderFactory.getInstance(); final PsiBuilder builder = factory.createBuilder(project, chameleon, chameleon.getElementType().getLanguage(), chameleon.getChars()); diff --git a/platform/lang-api/src/com/intellij/lang/WhitespacesAndCommentsProcessor.java b/platform/lang-api/src/com/intellij/lang/WhitespacesAndCommentsProcessor.java index 31a79b7b0f79..d9830d2992d7 100644 --- a/platform/lang-api/src/com/intellij/lang/WhitespacesAndCommentsProcessor.java +++ b/platform/lang-api/src/com/intellij/lang/WhitespacesAndCommentsProcessor.java @@ -27,6 +27,13 @@ import java.util.List; * @see com.intellij.lang.PsiBuilder.Marker#setCustomEdgeProcessors(WhitespacesAndCommentsProcessor, WhitespacesAndCommentsProcessor) */ public interface WhitespacesAndCommentsProcessor { + /** + * Provides an ability for the processor to get a text of any of given tokens. + */ + interface TokenTextGetter { + CharSequence get(int i); + } + /** *

Analyzes whitespace and comment tokens at element's edge and returns element's edge position relative to these tokens. * Value returned by left edge processor will be used as a pointer to a first token of element. @@ -41,7 +48,9 @@ public interface WhitespacesAndCommentsProcessor { * whitespace or comment after element's end. * * @param tokens sequence of whitespace and comment tokens at the element's edge. - * @return position of element's edge relative to given tokens + * @param atStreamEdge true if sequence of tokens is located at the beginning or the end of token stream. + * @param getter token text getter. + * @return position of element's edge relative to given tokens. */ - int process(List tokens); + int process(List tokens, boolean atStreamEdge, TokenTextGetter getter); } diff --git a/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java b/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java index de5fb9cb590a..260e5ce3e2b6 100644 --- a/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java +++ b/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java @@ -40,7 +40,10 @@ import com.intellij.psi.tree.*; import com.intellij.util.CharTable; import com.intellij.util.IncorrectOperationException; import com.intellij.util.ThreeState; -import com.intellij.util.containers.*; +import com.intellij.util.containers.CollectionFactory; +import com.intellij.util.containers.Convertor; +import com.intellij.util.containers.LimitedPool; +import com.intellij.util.containers.Stack; import com.intellij.util.diff.DiffTree; import com.intellij.util.diff.DiffTreeChangeBuilder; import com.intellij.util.diff.FlyweightCapableTreeStructure; @@ -104,13 +107,13 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { }); private static final WhitespacesAndCommentsProcessor DEFAULT_LEFT_EDGE_PROCESSOR = new WhitespacesAndCommentsProcessor() { - public int process(final List tokens) { + public int process(final List tokens, final boolean atStreamEdge, final TokenTextGetter getter) { return tokens.size(); } }; private static final WhitespacesAndCommentsProcessor DEFAULT_RIGHT_EDGE_PROCESSOR = new WhitespacesAndCommentsProcessor() { - public int process(final List tokens) { + public int process(final List tokens, final boolean atStreamEdge, final TokenTextGetter getter) { return 0; } }; @@ -827,23 +830,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { markTokenTypeChecked(); - for (int i = 1; i < myProduction.size() - 1; i++) { - final ProductionMarker item = myProduction.get(i); - - if (item instanceof StartMarker && ((StartMarker)item).myDoneMarker == null) { - LOG.error(UNBALANCED_MESSAGE); - } - - final int prevProductionLexIndex = myProduction.get(i - 1).myLexemeIndex; - int wsStartIndex = item.myLexemeIndex; - while (wsStartIndex > prevProductionLexIndex && whitespaceOrComment(myLexTypes[wsStartIndex - 1])) wsStartIndex--; - - int wsEndIndex = item.myLexemeIndex; - while (wsEndIndex < myLexemeCount && whitespaceOrComment(myLexTypes[wsEndIndex])) wsEndIndex++; - - final List wsTokens = CollectionFactory.arrayList(myLexTypes, wsStartIndex, wsEndIndex); - item.myLexemeIndex = wsStartIndex + item.myEdgeProcessor.process(wsTokens); - } + balanceWhiteSpaces(); rootMarker.firstChild = rootMarker.lastChild = rootMarker.next = null; StartMarker curNode = rootMarker; @@ -891,6 +878,33 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { return rootMarker; } + private void balanceWhiteSpaces() { + for (int i = 1; i < myProduction.size() - 1; i++) { + final ProductionMarker item = myProduction.get(i); + + if (item instanceof StartMarker && ((StartMarker)item).myDoneMarker == null) { + LOG.error(UNBALANCED_MESSAGE); + } + + final int prevProductionLexIndex = myProduction.get(i - 1).myLexemeIndex; + int idx = item.myLexemeIndex; + while (idx > prevProductionLexIndex && whitespaceOrComment(myLexTypes[idx - 1])) idx--; + final int wsStartIndex = idx; + + int wsEndIndex = item.myLexemeIndex; + while (wsEndIndex < myLexemeCount && whitespaceOrComment(myLexTypes[wsEndIndex])) wsEndIndex++; + + final List wsTokens = CollectionFactory.arrayList(myLexTypes, wsStartIndex, wsEndIndex); + final boolean atEnd = (wsStartIndex == 0 || wsEndIndex == myLexemeCount); + final WhitespacesAndCommentsProcessor.TokenTextGetter getter = new WhitespacesAndCommentsProcessor.TokenTextGetter() { + public CharSequence get(final int i) { + return myText.subSequence(myLexStarts[wsStartIndex + i], myLexStarts[wsStartIndex + i + 1]); + } + }; + item.myLexemeIndex = wsStartIndex + item.myEdgeProcessor.process(wsTokens, atEnd, getter); + } + } + private void bind(CompositeElement ast, StartMarker marker) { bind(ast, marker, marker.myLexemeIndex); } diff --git a/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java b/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java index 40ab0e96a1ac..eab175bb58cb 100644 --- a/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java +++ b/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java @@ -291,14 +291,14 @@ public class LightPsiBuilderTest { @Test public void testCustomEdgeProcessors() throws Exception { final WhitespacesAndCommentsProcessor leftEdgeProcessor = new WhitespacesAndCommentsProcessor() { - public int process(List tokens) { + public int process(List tokens, boolean atStreamEdge, TokenTextGetter getter) { int pos = tokens.size() - 1; while (tokens.get(pos) != COMMENT && pos > 0) pos--; return pos; } }; final WhitespacesAndCommentsProcessor rightEdgeProcessor = new WhitespacesAndCommentsProcessor() { - public int process(List tokens) { + public int process(List tokens, boolean atStreamEdge, TokenTextGetter getter) { int pos = 0; while (tokens.get(pos) != COMMENT && pos < tokens.size()-1) pos++; return pos + 1; diff --git a/platform/util/src/com/intellij/openapi/util/text/StringUtil.java b/platform/util/src/com/intellij/openapi/util/text/StringUtil.java index e48478b57f57..c3ca694a25c5 100644 --- a/platform/util/src/com/intellij/openapi/util/text/StringUtil.java +++ b/platform/util/src/com/intellij/openapi/util/text/StringUtil.java @@ -305,6 +305,7 @@ public class StringUtil { } else if (c == '\r') { if (i + 1 < text.length() && text.charAt(i + 1) == '\n') { + //noinspection AssignmentToForLoopParameter i++; count++; } @@ -316,6 +317,14 @@ public class StringUtil { return count; } + public static boolean containsLineBreak(@NotNull CharSequence text) { + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '\n' || c == '\r') return true; + } + return false; + } + public static int lineColToOffset(@NotNull CharSequence text, int line, int col) { int curLine = 0; int offset = 0;