MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Add comments and single quotes string literals to JSON grammar

Browse Source
This commit is contained in:
Mikhail Golubev
2014-09-03 22:09:26 +04:00
parent c12104546d
commit 33afe64888
10 changed files with 160 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
json/gen/com/intellij/json/JsonElementTypes.java
+2
View File
@@ -18,9 +18,11 @@ public interface JsonElementTypes {
IElementType STRING_LITERAL = new JsonElementType("STRING_LITERAL");
IElementType VALUE = new JsonElementType("VALUE");
IElementType BLOCK_COMMENT = new JsonTokenType("BLOCK_COMMENT");
IElementType COLON = new JsonTokenType(":");
IElementType COMMA = new JsonTokenType(",");
IElementType FALSE = new JsonTokenType("false");
IElementType LINE_COMMENT = new JsonTokenType("LINE_COMMENT");
IElementType L_BRACKET = new JsonTokenType("[");
IElementType L_CURLY = new JsonTokenType("{");
IElementType NULL = new JsonTokenType("null");
json/gen/com/intellij/json/_JsonLexer.flex
+18 -14
View File
@@ -22,27 +22,31 @@ EOL="\r"|"\n"|"\r\n"
LINE_WS=[\ \t\f]
WHITE_SPACE=({LINE_WS}|{EOL})+
STRING=\"([^\\\"\r\n]|\\[^\r\n])*\"?
LINE_COMMENT="//".*
BLOCK_COMMENT="/"\*([^*]|\*+[^*/])*(\*+"/")?
STRING=\"([^\\\"\r\n]|\\[^\r\n])*\"?|'([^\\\"\r\n]|\\[^\r\n])*'?
NUMBER=-?[0-9]+(\.[0-9]+([eE][+-]?[0-9]+)?)?
TEXT=[a-zA-Z_0-9]+
%%
<YYINITIAL> {
{WHITE_SPACE} { return com.intellij.psi.TokenType.WHITE_SPACE; }
{WHITE_SPACE} { return com.intellij.psi.TokenType.WHITE_SPACE; }
"{" { return L_CURLY; }
"}" { return R_CURLY; }
"[" { return L_BRACKET; }
"]" { return R_BRACKET; }
"," { return COMMA; }
":" { return COLON; }
"true" { return TRUE; }
"false" { return FALSE; }
"null" { return NULL; }
"{" { return L_CURLY; }
"}" { return R_CURLY; }
"[" { return L_BRACKET; }
"]" { return R_BRACKET; }
"," { return COMMA; }
":" { return COLON; }
"true" { return TRUE; }
"false" { return FALSE; }
"null" { return NULL; }
{STRING} { return STRING; }
{NUMBER} { return NUMBER; }
{TEXT} { return TEXT; }
{LINE_COMMENT} { return LINE_COMMENT; }
{BLOCK_COMMENT} { return BLOCK_COMMENT; }
{STRING} { return STRING; }
{NUMBER} { return NUMBER; }
{TEXT} { return TEXT; }
[^] { return com.intellij.psi.TokenType.BAD_CHARACTER; }
}
json/gen/com/intellij/json/_JsonLexer.java
+77 -63
View File
@@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.4.3 on 7/15/14 2:07 PM */
/* The following code was generated by JFlex 1.4.3 on 9/3/14 6:05 PM */
package com.intellij.json;
import com.intellij.lexer.*;
@@ -9,8 +9,8 @@ import static com.intellij.json.JsonElementTypes.*;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 7/15/14 2:07 PM from the specification file
* <tt>/home/east825/develop/repos/IDEA-branch/community/plugins/json/gen/com/intellij/json/_JsonLexer.flex</tt>
* on 9/3/14 6:05 PM from the specification file
* <tt>/home/east825/develop/repos/IDEA/community/json/gen/com/intellij/json/_JsonLexer.flex</tt>
*/
public class _JsonLexer implements FlexLexer {
/** initial size of the lookahead buffer */
@@ -33,11 +33,11 @@ public class _JsonLexer implements FlexLexer {
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
"\11\0\1\2\1\1\1\0\1\2\1\1\22\0\1\2\1\0\1\3"+
"\10\0\1\11\1\17\1\5\1\7\1\0\12\6\1\20\6\0\4\12"+
"\1\10\25\12\1\15\1\4\1\16\1\0\1\12\1\0\1\26\3\12"+
"\1\24\1\25\5\12\1\27\1\12\1\31\3\12\1\22\1\30\1\21"+
"\1\23\5\12\1\13\1\0\1\14\uff82\0";
"\11\0\1\3\1\2\1\0\1\3\1\1\22\0\1\3\1\0\1\6"+
"\4\0\1\10\2\0\1\5\1\15\1\23\1\11\1\13\1\4\12\12"+
"\1\24\6\0\4\16\1\14\25\16\1\21\1\7\1\22\1\0\1\16"+
"\1\0\1\32\3\16\1\30\1\31\5\16\1\33\1\16\1\35\3\16"+
"\1\26\1\34\1\25\1\27\5\16\1\17\1\0\1\20\uff82\0";
/**
* Translates characters to character classes
@@ -50,13 +50,14 @@ public class _JsonLexer implements FlexLexer {
private static final int [] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
"\1\0\1\1\1\2\1\3\1\1\1\4\1\5\1\6"+
"\1\7\1\10\1\11\1\12\1\13\3\5\1\3\1\0"+
"\1\4\1\0\3\5\1\4\3\5\1\0\1\14\1\5"+
"\1\15\1\0\1\4\1\16";
"\1\0\1\1\1\2\1\1\2\3\1\1\1\4\1\5"+
"\1\6\1\7\1\10\1\11\1\12\1\13\3\5\1\14"+
"\1\15\1\3\2\0\1\4\1\0\3\5\1\0\1\4"+
"\3\5\1\15\1\0\1\16\1\5\1\17\1\0\1\4"+
"\1\20";
private static int [] zzUnpackAction() {
int [] result = new int[34];
int [] result = new int[41];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
@@ -81,14 +82,15 @@ public class _JsonLexer implements FlexLexer {
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
"\0\0\0\32\0\64\0\116\0\150\0\202\0\234\0\32"+
"\0\32\0\32\0\32\0\32\0\32\0\266\0\320\0\352"+
"\0\32\0\u0104\0\u011e\0\u0138\0\u0152\0\u016c\0\u0186\0\u01a0"+
"\0\u01ba\0\u01d4\0\u01ee\0\u0208\0\234\0\u0222\0\234\0\u023c"+
"\0\u023c\0\234";
"\0\0\0\36\0\74\0\132\0\170\0\226\0\264\0\322"+
"\0\360\0\36\0\36\0\36\0\36\0\36\0\36\0\u010e"+
"\0\u012c\0\u014a\0\u0168\0\u0186\0\36\0\u01a4\0\u01c2\0\u01e0"+
"\0\u01fe\0\u021c\0\u023a\0\u0258\0\u0276\0\u0294\0\u02b2\0\u02d0"+
"\0\u02ee\0\36\0\u030c\0\360\0\u032a\0\360\0\u0348\0\u0348"+
"\0\360";
private static int [] zzUnpackRowMap() {
int [] result = new int[34];
int [] result = new int[41];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
@@ -111,31 +113,34 @@ public class _JsonLexer implements FlexLexer {
private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
"\1\2\2\3\1\4\1\2\1\5\1\6\1\2\1\7"+
"\1\2\1\7\1\10\1\11\1\12\1\13\1\14\1\15"+
"\1\16\3\7\1\17\3\7\1\20\33\0\2\3\27\0"+
"\1\4\1\0\1\4\1\21\1\22\25\4\6\0\1\23"+
"\31\0\1\6\1\24\1\7\1\0\1\7\6\0\11\7"+
"\6\0\1\7\1\0\1\7\1\0\1\7\6\0\11\7"+
"\6\0\1\7\1\0\1\7\1\0\1\7\6\0\1\7"+
"\1\25\7\7\6\0\1\7\1\0\1\7\1\0\1\7"+
"\6\0\5\7\1\26\3\7\6\0\1\7\1\0\1\7"+
"\1\0\1\7\6\0\2\7\1\27\6\7\1\4\1\0"+
"\30\4\6\0\1\23\1\24\30\0\1\30\31\0\1\7"+
"\1\0\1\7\1\0\1\7\6\0\2\7\1\31\6\7"+
"\6\0\1\7\1\0\1\7\1\0\1\7\6\0\6\7"+
"\1\32\2\7\6\0\1\7\1\0\1\7\1\0\1\7"+
"\6\0\6\7\1\33\2\7\6\0\1\30\1\0\1\34"+
"\13\0\1\34\13\0\1\7\1\0\1\7\1\0\1\7"+
"\6\0\3\7\1\35\5\7\6\0\1\7\1\0\1\7"+
"\1\0\1\7\6\0\7\7\1\36\1\7\6\0\1\7"+
"\1\0\1\7\1\0\1\7\6\0\6\7\1\37\2\7"+
"\5\0\1\40\1\41\2\0\1\40\26\0\1\7\1\0"+
"\1\7\1\0\1\7\6\0\3\7\1\42\5\7\6\0"+
"\1\41\23\0";
"\1\2\3\3\1\4\1\2\1\5\1\2\1\6\1\7"+
"\1\10\1\2\1\11\1\2\1\11\1\12\1\13\1\14"+
"\1\15\1\16\1\17\1\20\3\11\1\21\3\11\1\22"+
"\37\0\3\3\36\0\1\23\1\24\30\0\1\5\2\0"+
"\3\5\1\25\1\26\26\5\1\6\2\0\3\6\1\0"+
"\1\27\26\6\12\0\1\30\35\0\1\10\1\31\1\11"+
"\1\0\1\11\6\0\11\11\12\0\1\11\1\0\1\11"+
"\1\0\1\11\6\0\11\11\12\0\1\11\1\0\1\11"+
"\1\0\1\11\6\0\1\11\1\32\7\11\12\0\1\11"+
"\1\0\1\11\1\0\1\11\6\0\5\11\1\33\3\11"+
"\12\0\1\11\1\0\1\11\1\0\1\11\6\0\2\11"+
"\1\34\6\11\2\23\1\0\33\23\5\24\1\35\30\24"+
"\1\5\2\0\33\5\1\6\2\0\33\6\12\0\1\30"+
"\1\31\34\0\1\36\35\0\1\11\1\0\1\11\1\0"+
"\1\11\6\0\2\11\1\37\6\11\12\0\1\11\1\0"+
"\1\11\1\0\1\11\6\0\6\11\1\40\2\11\12\0"+
"\1\11\1\0\1\11\1\0\1\11\6\0\6\11\1\41"+
"\2\11\4\24\1\42\1\35\30\24\12\0\1\36\1\0"+
"\1\43\13\0\1\43\17\0\1\11\1\0\1\11\1\0"+
"\1\11\6\0\3\11\1\44\5\11\12\0\1\11\1\0"+
"\1\11\1\0\1\11\6\0\7\11\1\45\1\11\12\0"+
"\1\11\1\0\1\11\1\0\1\11\6\0\6\11\1\46"+
"\2\11\11\0\1\47\1\50\2\0\1\47\32\0\1\11"+
"\1\0\1\11\1\0\1\11\6\0\3\11\1\51\5\11"+
"\12\0\1\50\23\0";
private static int [] zzUnpackTrans() {
int [] result = new int[598];
int [] result = new int[870];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
@@ -176,11 +181,12 @@ public class _JsonLexer implements FlexLexer {
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
"\1\0\1\11\5\1\6\11\3\1\1\11\1\0\1\1"+
"\1\0\7\1\1\0\3\1\1\0\2\1";
"\1\0\1\11\7\1\6\11\5\1\1\11\2\0\1\1"+
"\1\0\3\1\1\0\4\1\1\11\1\0\3\1\1\0"+
"\2\1";
private static int [] zzUnpackAttribute() {
int [] result = new int[34];
int [] result = new int[41];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
@@ -265,7 +271,7 @@ public class _JsonLexer implements FlexLexer {
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 92) {
while (i < 98) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
@@ -478,59 +484,67 @@ public class _JsonLexer implements FlexLexer {
case 7:
{ return R_CURLY;
}
case 15: break;
case 17: break;
case 6:
{ return L_CURLY;
}
case 16: break;
case 13:
case 18: break;
case 15:
{ return NULL;
}
case 17: break;
case 19: break;
case 8:
{ return L_BRACKET;
}
case 18: break;
case 20: break;
case 12:
{ return LINE_COMMENT;
}
case 21: break;
case 10:
{ return COMMA;
}
case 19: break;
case 22: break;
case 2:
{ return com.intellij.psi.TokenType.WHITE_SPACE;
}
case 20: break;
case 23: break;
case 5:
{ return TEXT;
}
case 21: break;
case 24: break;
case 1:
{ return com.intellij.psi.TokenType.BAD_CHARACTER;
}
case 22: break;
case 12:
case 25: break;
case 14:
{ return TRUE;
}
case 23: break;
case 26: break;
case 11:
{ return COLON;
}
case 24: break;
case 27: break;
case 3:
{ return STRING;
}
case 25: break;
case 28: break;
case 4:
{ return NUMBER;
}
case 26: break;
case 14:
case 29: break;
case 16:
{ return FALSE;
}
case 27: break;
case 30: break;
case 13:
{ return BLOCK_COMMENT;
}
case 31: break;
case 9:
{ return R_BRACKET;
}
case 28: break;
case 32: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
json/src/com/intellij/json/JsonParserDefinition.java
+2 -2
View File
@@ -55,7 +55,7 @@ public class JsonParserDefinition implements ParserDefinition {
@NotNull
@Override
public TokenSet getCommentTokens() {
return TokenSet.EMPTY;
return TokenSet.create(BLOCK_COMMENT, LINE_COMMENT);
}
@NotNull
@@ -67,7 +67,7 @@ public class JsonParserDefinition implements ParserDefinition {
@NotNull
@Override
public PsiElement createElement(ASTNode astNode) {
return JsonElementTypes.Factory.createElement(astNode);
return Factory.createElement(astNode);
}
@Override
json/src/json.bnf
+5 -3
View File
@@ -19,16 +19,18 @@
R_CURLY='}'
L_BRACKET='['
R_BRACKET=']'
COMMA=','
COLON=':'
// COMMENT='regexp:"//.*|/\*.*?\*/"'
LINE_COMMENT='regexp://.*'
// "/*" ([^*]|\*+[^*/])* (\*+"/")?
BLOCK_COMMENT='regexp:/\*([^*]|\*+[^*/])*(\*+/)?'
// else /\*(?:[^*]|\*[^/])*\*+/
// unclosed string literal matches till the line's end
// any escape sequences included, illegal escapes are indicated by SyntaxHighlighter
// and JsonStringLiteralAnnotator
STRING='regexp:"([^\\"\r\n]|\\[^\r\n])*"?'
STRING="regexp:\"([^\\\"\r\n]|\\[^\r\n])*\"?|'([^\\\"\r\n]|\\[^\r\n])*'?"
// STRING='regexp:"([^\\"\r\n]|\\([\\"/bfnrt]|u[a-fA-F0-9]{4}))*"?'
NUMBER='regexp:-?\d+(\.\d+([eE][+-]?\d+)?)?'
json/tests/test/com/intellij/json/JsonParsingTest.java
+9
View File
@@ -54,6 +54,15 @@ public class JsonParsingTest extends ParsingTestCase {
// doTest();
//}
public void testComments() {
doTest();
}
public void testStringLiterals() {
doTest();
}
private void doTest() {
doTest(true);
}
json/tests/testData/psi/Comments.json
+15
View File
@@ -0,0 +1,15 @@
// End of line comment
{
/**/
/***/
/*** */
/** **/
/* **/
/*a*/ /*b*/
/* C-style block comment */
/* May span
multiple
lines
*/
}
/* And be incomplete
json/tests/testData/psi/Comments.txt
+27
View File
@@ -0,0 +1,27 @@
JsonFile: Comments.json
PsiComment(LINE_COMMENT)('// End of line comment')
PsiWhiteSpace('\n')
JsonObject
PsiElement({)('{')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/**/')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/***/')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/*** */')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/** **/')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/* **/')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/*a*/')
PsiWhiteSpace(' ')
PsiComment(BLOCK_COMMENT)('/*b*/')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/* C-style block comment */')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/* May span\nmultiple\nlines\n */')
PsiWhiteSpace('\n')
PsiElement(})('}')
PsiWhiteSpace('\n')
PsiComment(BLOCK_COMMENT)('/* And be incomplete')
json/tests/testData/psi/StringLiterals.json
+4
View File
@@ -0,0 +1,4 @@
{
'foo': "bar",
"baz": 'quux'
}
tools/lexer/jflex-1.4/bin/jflex
+1 -1
View File
@@ -1,4 +1,4 @@
#! /bin/bash
#!/bin/bash
#
# JFlex start script $Revision$
#