From 33afe64888a2981d0664e7cc870c98dab541cffc Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Wed, 3 Sep 2014 18:08:56 +0400 Subject: [PATCH] Add comments and single quotes string literals to JSON grammar --- .../com/intellij/json/JsonElementTypes.java | 2 + json/gen/com/intellij/json/_JsonLexer.flex | 32 ++-- json/gen/com/intellij/json/_JsonLexer.java | 140 ++++++++++-------- .../intellij/json/JsonParserDefinition.java | 4 +- json/src/json.bnf | 8 +- .../com/intellij/json/JsonParsingTest.java | 9 ++ json/tests/testData/psi/Comments.json | 15 ++ json/tests/testData/psi/Comments.txt | 27 ++++ json/tests/testData/psi/StringLiterals.json | 4 + tools/lexer/jflex-1.4/bin/jflex | 2 +- 10 files changed, 160 insertions(+), 83 deletions(-) create mode 100644 json/tests/testData/psi/Comments.json create mode 100644 json/tests/testData/psi/Comments.txt create mode 100644 json/tests/testData/psi/StringLiterals.json diff --git a/json/gen/com/intellij/json/JsonElementTypes.java b/json/gen/com/intellij/json/JsonElementTypes.java index 602797280783..aa3d749ce7d2 100644 --- a/json/gen/com/intellij/json/JsonElementTypes.java +++ b/json/gen/com/intellij/json/JsonElementTypes.java @@ -18,9 +18,11 @@ public interface JsonElementTypes { IElementType STRING_LITERAL = new JsonElementType("STRING_LITERAL"); IElementType VALUE = new JsonElementType("VALUE"); + IElementType BLOCK_COMMENT = new JsonTokenType("BLOCK_COMMENT"); IElementType COLON = new JsonTokenType(":"); IElementType COMMA = new JsonTokenType(","); IElementType FALSE = new JsonTokenType("false"); + IElementType LINE_COMMENT = new JsonTokenType("LINE_COMMENT"); IElementType L_BRACKET = new JsonTokenType("["); IElementType L_CURLY = new JsonTokenType("{"); IElementType NULL = new JsonTokenType("null"); diff --git a/json/gen/com/intellij/json/_JsonLexer.flex b/json/gen/com/intellij/json/_JsonLexer.flex index 0083719fb629..af5784b60498 100644 --- a/json/gen/com/intellij/json/_JsonLexer.flex +++ b/json/gen/com/intellij/json/_JsonLexer.flex @@ -22,27 +22,31 @@ EOL="\r"|"\n"|"\r\n" LINE_WS=[\ \t\f] WHITE_SPACE=({LINE_WS}|{EOL})+ -STRING=\"([^\\\"\r\n]|\\[^\r\n])*\"? +LINE_COMMENT="//".* +BLOCK_COMMENT="/"\*([^*]|\*+[^*/])*(\*+"/")? +STRING=\"([^\\\"\r\n]|\\[^\r\n])*\"?|'([^\\\"\r\n]|\\[^\r\n])*'? NUMBER=-?[0-9]+(\.[0-9]+([eE][+-]?[0-9]+)?)? TEXT=[a-zA-Z_0-9]+ %% { - {WHITE_SPACE} { return com.intellij.psi.TokenType.WHITE_SPACE; } + {WHITE_SPACE} { return com.intellij.psi.TokenType.WHITE_SPACE; } - "{" { return L_CURLY; } - "}" { return R_CURLY; } - "[" { return L_BRACKET; } - "]" { return R_BRACKET; } - "," { return COMMA; } - ":" { return COLON; } - "true" { return TRUE; } - "false" { return FALSE; } - "null" { return NULL; } + "{" { return L_CURLY; } + "}" { return R_CURLY; } + "[" { return L_BRACKET; } + "]" { return R_BRACKET; } + "," { return COMMA; } + ":" { return COLON; } + "true" { return TRUE; } + "false" { return FALSE; } + "null" { return NULL; } - {STRING} { return STRING; } - {NUMBER} { return NUMBER; } - {TEXT} { return TEXT; } + {LINE_COMMENT} { return LINE_COMMENT; } + {BLOCK_COMMENT} { return BLOCK_COMMENT; } + {STRING} { return STRING; } + {NUMBER} { return NUMBER; } + {TEXT} { return TEXT; } [^] { return com.intellij.psi.TokenType.BAD_CHARACTER; } } diff --git a/json/gen/com/intellij/json/_JsonLexer.java b/json/gen/com/intellij/json/_JsonLexer.java index 1cf42f524aa8..396b5eb6e571 100644 --- a/json/gen/com/intellij/json/_JsonLexer.java +++ b/json/gen/com/intellij/json/_JsonLexer.java @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.3 on 7/15/14 2:07 PM */ +/* The following code was generated by JFlex 1.4.3 on 9/3/14 6:05 PM */ package com.intellij.json; import com.intellij.lexer.*; @@ -9,8 +9,8 @@ import static com.intellij.json.JsonElementTypes.*; /** * This class is a scanner generated by * JFlex 1.4.3 - * on 7/15/14 2:07 PM from the specification file - * /home/east825/develop/repos/IDEA-branch/community/plugins/json/gen/com/intellij/json/_JsonLexer.flex + * on 9/3/14 6:05 PM from the specification file + * /home/east825/develop/repos/IDEA/community/json/gen/com/intellij/json/_JsonLexer.flex */ public class _JsonLexer implements FlexLexer { /** initial size of the lookahead buffer */ @@ -33,11 +33,11 @@ public class _JsonLexer implements FlexLexer { * Translates characters to character classes */ private static final String ZZ_CMAP_PACKED = - "\11\0\1\2\1\1\1\0\1\2\1\1\22\0\1\2\1\0\1\3"+ - "\10\0\1\11\1\17\1\5\1\7\1\0\12\6\1\20\6\0\4\12"+ - "\1\10\25\12\1\15\1\4\1\16\1\0\1\12\1\0\1\26\3\12"+ - "\1\24\1\25\5\12\1\27\1\12\1\31\3\12\1\22\1\30\1\21"+ - "\1\23\5\12\1\13\1\0\1\14\uff82\0"; + "\11\0\1\3\1\2\1\0\1\3\1\1\22\0\1\3\1\0\1\6"+ + "\4\0\1\10\2\0\1\5\1\15\1\23\1\11\1\13\1\4\12\12"+ + "\1\24\6\0\4\16\1\14\25\16\1\21\1\7\1\22\1\0\1\16"+ + "\1\0\1\32\3\16\1\30\1\31\5\16\1\33\1\16\1\35\3\16"+ + "\1\26\1\34\1\25\1\27\5\16\1\17\1\0\1\20\uff82\0"; /** * Translates characters to character classes @@ -50,13 +50,14 @@ public class _JsonLexer implements FlexLexer { private static final int [] ZZ_ACTION = zzUnpackAction(); private static final String ZZ_ACTION_PACKED_0 = - "\1\0\1\1\1\2\1\3\1\1\1\4\1\5\1\6"+ - "\1\7\1\10\1\11\1\12\1\13\3\5\1\3\1\0"+ - "\1\4\1\0\3\5\1\4\3\5\1\0\1\14\1\5"+ - "\1\15\1\0\1\4\1\16"; + "\1\0\1\1\1\2\1\1\2\3\1\1\1\4\1\5"+ + "\1\6\1\7\1\10\1\11\1\12\1\13\3\5\1\14"+ + "\1\15\1\3\2\0\1\4\1\0\3\5\1\0\1\4"+ + "\3\5\1\15\1\0\1\16\1\5\1\17\1\0\1\4"+ + "\1\20"; private static int [] zzUnpackAction() { - int [] result = new int[34]; + int [] result = new int[41]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; @@ -81,14 +82,15 @@ public class _JsonLexer implements FlexLexer { private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\32\0\64\0\116\0\150\0\202\0\234\0\32"+ - "\0\32\0\32\0\32\0\32\0\32\0\266\0\320\0\352"+ - "\0\32\0\u0104\0\u011e\0\u0138\0\u0152\0\u016c\0\u0186\0\u01a0"+ - "\0\u01ba\0\u01d4\0\u01ee\0\u0208\0\234\0\u0222\0\234\0\u023c"+ - "\0\u023c\0\234"; + "\0\0\0\36\0\74\0\132\0\170\0\226\0\264\0\322"+ + "\0\360\0\36\0\36\0\36\0\36\0\36\0\36\0\u010e"+ + "\0\u012c\0\u014a\0\u0168\0\u0186\0\36\0\u01a4\0\u01c2\0\u01e0"+ + "\0\u01fe\0\u021c\0\u023a\0\u0258\0\u0276\0\u0294\0\u02b2\0\u02d0"+ + "\0\u02ee\0\36\0\u030c\0\360\0\u032a\0\360\0\u0348\0\u0348"+ + "\0\360"; private static int [] zzUnpackRowMap() { - int [] result = new int[34]; + int [] result = new int[41]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; @@ -111,31 +113,34 @@ public class _JsonLexer implements FlexLexer { private static final int [] ZZ_TRANS = zzUnpackTrans(); private static final String ZZ_TRANS_PACKED_0 = - "\1\2\2\3\1\4\1\2\1\5\1\6\1\2\1\7"+ - "\1\2\1\7\1\10\1\11\1\12\1\13\1\14\1\15"+ - "\1\16\3\7\1\17\3\7\1\20\33\0\2\3\27\0"+ - "\1\4\1\0\1\4\1\21\1\22\25\4\6\0\1\23"+ - "\31\0\1\6\1\24\1\7\1\0\1\7\6\0\11\7"+ - "\6\0\1\7\1\0\1\7\1\0\1\7\6\0\11\7"+ - "\6\0\1\7\1\0\1\7\1\0\1\7\6\0\1\7"+ - "\1\25\7\7\6\0\1\7\1\0\1\7\1\0\1\7"+ - "\6\0\5\7\1\26\3\7\6\0\1\7\1\0\1\7"+ - "\1\0\1\7\6\0\2\7\1\27\6\7\1\4\1\0"+ - "\30\4\6\0\1\23\1\24\30\0\1\30\31\0\1\7"+ - "\1\0\1\7\1\0\1\7\6\0\2\7\1\31\6\7"+ - "\6\0\1\7\1\0\1\7\1\0\1\7\6\0\6\7"+ - "\1\32\2\7\6\0\1\7\1\0\1\7\1\0\1\7"+ - "\6\0\6\7\1\33\2\7\6\0\1\30\1\0\1\34"+ - "\13\0\1\34\13\0\1\7\1\0\1\7\1\0\1\7"+ - "\6\0\3\7\1\35\5\7\6\0\1\7\1\0\1\7"+ - "\1\0\1\7\6\0\7\7\1\36\1\7\6\0\1\7"+ - "\1\0\1\7\1\0\1\7\6\0\6\7\1\37\2\7"+ - "\5\0\1\40\1\41\2\0\1\40\26\0\1\7\1\0"+ - "\1\7\1\0\1\7\6\0\3\7\1\42\5\7\6\0"+ - "\1\41\23\0"; + "\1\2\3\3\1\4\1\2\1\5\1\2\1\6\1\7"+ + "\1\10\1\2\1\11\1\2\1\11\1\12\1\13\1\14"+ + "\1\15\1\16\1\17\1\20\3\11\1\21\3\11\1\22"+ + "\37\0\3\3\36\0\1\23\1\24\30\0\1\5\2\0"+ + "\3\5\1\25\1\26\26\5\1\6\2\0\3\6\1\0"+ + "\1\27\26\6\12\0\1\30\35\0\1\10\1\31\1\11"+ + "\1\0\1\11\6\0\11\11\12\0\1\11\1\0\1\11"+ + "\1\0\1\11\6\0\11\11\12\0\1\11\1\0\1\11"+ + "\1\0\1\11\6\0\1\11\1\32\7\11\12\0\1\11"+ + "\1\0\1\11\1\0\1\11\6\0\5\11\1\33\3\11"+ + "\12\0\1\11\1\0\1\11\1\0\1\11\6\0\2\11"+ + "\1\34\6\11\2\23\1\0\33\23\5\24\1\35\30\24"+ + "\1\5\2\0\33\5\1\6\2\0\33\6\12\0\1\30"+ + "\1\31\34\0\1\36\35\0\1\11\1\0\1\11\1\0"+ + "\1\11\6\0\2\11\1\37\6\11\12\0\1\11\1\0"+ + "\1\11\1\0\1\11\6\0\6\11\1\40\2\11\12\0"+ + "\1\11\1\0\1\11\1\0\1\11\6\0\6\11\1\41"+ + "\2\11\4\24\1\42\1\35\30\24\12\0\1\36\1\0"+ + "\1\43\13\0\1\43\17\0\1\11\1\0\1\11\1\0"+ + "\1\11\6\0\3\11\1\44\5\11\12\0\1\11\1\0"+ + "\1\11\1\0\1\11\6\0\7\11\1\45\1\11\12\0"+ + "\1\11\1\0\1\11\1\0\1\11\6\0\6\11\1\46"+ + "\2\11\11\0\1\47\1\50\2\0\1\47\32\0\1\11"+ + "\1\0\1\11\1\0\1\11\6\0\3\11\1\51\5\11"+ + "\12\0\1\50\23\0"; private static int [] zzUnpackTrans() { - int [] result = new int[598]; + int [] result = new int[870]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; @@ -176,11 +181,12 @@ public class _JsonLexer implements FlexLexer { private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); private static final String ZZ_ATTRIBUTE_PACKED_0 = - "\1\0\1\11\5\1\6\11\3\1\1\11\1\0\1\1"+ - "\1\0\7\1\1\0\3\1\1\0\2\1"; + "\1\0\1\11\7\1\6\11\5\1\1\11\2\0\1\1"+ + "\1\0\3\1\1\0\4\1\1\11\1\0\3\1\1\0"+ + "\2\1"; private static int [] zzUnpackAttribute() { - int [] result = new int[34]; + int [] result = new int[41]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; @@ -265,7 +271,7 @@ public class _JsonLexer implements FlexLexer { char [] map = new char[0x10000]; int i = 0; /* index in packed string */ int j = 0; /* index in unpacked array */ - while (i < 92) { + while (i < 98) { int count = packed.charAt(i++); char value = packed.charAt(i++); do map[j++] = value; while (--count > 0); @@ -478,59 +484,67 @@ public class _JsonLexer implements FlexLexer { case 7: { return R_CURLY; } - case 15: break; + case 17: break; case 6: { return L_CURLY; } - case 16: break; - case 13: + case 18: break; + case 15: { return NULL; } - case 17: break; + case 19: break; case 8: { return L_BRACKET; } - case 18: break; + case 20: break; + case 12: + { return LINE_COMMENT; + } + case 21: break; case 10: { return COMMA; } - case 19: break; + case 22: break; case 2: { return com.intellij.psi.TokenType.WHITE_SPACE; } - case 20: break; + case 23: break; case 5: { return TEXT; } - case 21: break; + case 24: break; case 1: { return com.intellij.psi.TokenType.BAD_CHARACTER; } - case 22: break; - case 12: + case 25: break; + case 14: { return TRUE; } - case 23: break; + case 26: break; case 11: { return COLON; } - case 24: break; + case 27: break; case 3: { return STRING; } - case 25: break; + case 28: break; case 4: { return NUMBER; } - case 26: break; - case 14: + case 29: break; + case 16: { return FALSE; } - case 27: break; + case 30: break; + case 13: + { return BLOCK_COMMENT; + } + case 31: break; case 9: { return R_BRACKET; } - case 28: break; + case 32: break; default: if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { zzAtEOF = true; diff --git a/json/src/com/intellij/json/JsonParserDefinition.java b/json/src/com/intellij/json/JsonParserDefinition.java index 506072a36551..685cdea9e262 100644 --- a/json/src/com/intellij/json/JsonParserDefinition.java +++ b/json/src/com/intellij/json/JsonParserDefinition.java @@ -55,7 +55,7 @@ public class JsonParserDefinition implements ParserDefinition { @NotNull @Override public TokenSet getCommentTokens() { - return TokenSet.EMPTY; + return TokenSet.create(BLOCK_COMMENT, LINE_COMMENT); } @NotNull @@ -67,7 +67,7 @@ public class JsonParserDefinition implements ParserDefinition { @NotNull @Override public PsiElement createElement(ASTNode astNode) { - return JsonElementTypes.Factory.createElement(astNode); + return Factory.createElement(astNode); } @Override diff --git a/json/src/json.bnf b/json/src/json.bnf index 6cdab08b01e9..b4e8b334bb9f 100644 --- a/json/src/json.bnf +++ b/json/src/json.bnf @@ -19,16 +19,18 @@ R_CURLY='}' L_BRACKET='[' R_BRACKET=']' - + COMMA=',' COLON=':' - // COMMENT='regexp:"//.*|/\*.*?\*/"' + LINE_COMMENT='regexp://.*' + // "/*" ([^*]|\*+[^*/])* (\*+"/")? + BLOCK_COMMENT='regexp:/\*([^*]|\*+[^*/])*(\*+/)?' // else /\*(?:[^*]|\*[^/])*\*+/ // unclosed string literal matches till the line's end // any escape sequences included, illegal escapes are indicated by SyntaxHighlighter // and JsonStringLiteralAnnotator - STRING='regexp:"([^\\"\r\n]|\\[^\r\n])*"?' + STRING="regexp:\"([^\\\"\r\n]|\\[^\r\n])*\"?|'([^\\\"\r\n]|\\[^\r\n])*'?" // STRING='regexp:"([^\\"\r\n]|\\([\\"/bfnrt]|u[a-fA-F0-9]{4}))*"?' NUMBER='regexp:-?\d+(\.\d+([eE][+-]?\d+)?)?' diff --git a/json/tests/test/com/intellij/json/JsonParsingTest.java b/json/tests/test/com/intellij/json/JsonParsingTest.java index 5fc7c9902b98..ab48ee1f4ab2 100644 --- a/json/tests/test/com/intellij/json/JsonParsingTest.java +++ b/json/tests/test/com/intellij/json/JsonParsingTest.java @@ -54,6 +54,15 @@ public class JsonParsingTest extends ParsingTestCase { // doTest(); //} + + public void testComments() { + doTest(); + } + + public void testStringLiterals() { + doTest(); + } + private void doTest() { doTest(true); } diff --git a/json/tests/testData/psi/Comments.json b/json/tests/testData/psi/Comments.json new file mode 100644 index 000000000000..60ad9bf4e447 --- /dev/null +++ b/json/tests/testData/psi/Comments.json @@ -0,0 +1,15 @@ +// End of line comment +{ +/**/ +/***/ +/*** */ +/** **/ +/* **/ +/*a*/ /*b*/ +/* C-style block comment */ +/* May span +multiple +lines + */ +} +/* And be incomplete \ No newline at end of file diff --git a/json/tests/testData/psi/Comments.txt b/json/tests/testData/psi/Comments.txt new file mode 100644 index 000000000000..7225a522186e --- /dev/null +++ b/json/tests/testData/psi/Comments.txt @@ -0,0 +1,27 @@ +JsonFile: Comments.json + PsiComment(LINE_COMMENT)('// End of line comment') + PsiWhiteSpace('\n') + JsonObject + PsiElement({)('{') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/**/') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/***/') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/*** */') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/** **/') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/* **/') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/*a*/') + PsiWhiteSpace(' ') + PsiComment(BLOCK_COMMENT)('/*b*/') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/* C-style block comment */') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/* May span\nmultiple\nlines\n */') + PsiWhiteSpace('\n') + PsiElement(})('}') + PsiWhiteSpace('\n') + PsiComment(BLOCK_COMMENT)('/* And be incomplete') \ No newline at end of file diff --git a/json/tests/testData/psi/StringLiterals.json b/json/tests/testData/psi/StringLiterals.json new file mode 100644 index 000000000000..e55eaff73c3b --- /dev/null +++ b/json/tests/testData/psi/StringLiterals.json @@ -0,0 +1,4 @@ +{ + 'foo': "bar", + "baz": 'quux' +} \ No newline at end of file diff --git a/tools/lexer/jflex-1.4/bin/jflex b/tools/lexer/jflex-1.4/bin/jflex index d6ad789e2490..65cb93efbbbb 100755 --- a/tools/lexer/jflex-1.4/bin/jflex +++ b/tools/lexer/jflex-1.4/bin/jflex @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash # # JFlex start script $Revision$ #