MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

external build: honor encoding specified in xml files (IDEA-97558)

Browse Source
This commit is contained in:
nik
2012-12-18 11:29:46 +04:00
parent 0548b82842
commit 837245dcb3
9 changed files with 250 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps/model-impl/src/org/jetbrains/jps/model/impl/JpsEncodingProjectConfigurationImpl.java
+25
View File
@@ -15,7 +15,12 @@
*/
package org.jetbrains.jps.model.impl;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.text.XmlCharsetDetector;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.model.JpsElementChildRole;
@@ -27,6 +32,7 @@ import org.jetbrains.jps.model.ex.JpsElementChildRoleBase;
import org.jetbrains.jps.util.JpsPathUtil;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -36,7 +42,9 @@ import java.util.Map;
*/
public class JpsEncodingProjectConfigurationImpl extends JpsElementBase<JpsEncodingProjectConfigurationImpl>
implements JpsEncodingProjectConfiguration {
private static final Logger LOG = Logger.getInstance(JpsEncodingProjectConfigurationImpl.class);
public static final JpsElementChildRole<JpsEncodingProjectConfiguration> ROLE = JpsElementChildRoleBase.create("encoding configuration");
private static final String XML_NAME_SUFFIX = ".xml";
private final Map<String, String> myUrlToEncoding = new HashMap<String, String>();
private final String myProjectEncoding;
@@ -48,6 +56,18 @@ public class JpsEncodingProjectConfigurationImpl extends JpsElementBase<JpsEncod
@Nullable
@Override
public String getEncoding(@NotNull File file) {
if (isXmlFile(file)) {
try {
String encoding = XmlCharsetDetector.extractXmlEncodingFromProlog(FileUtil.loadFileBytes(file));
if (encoding != null) {
return encoding;
}
}
catch (IOException e) {
LOG.info("Cannot detect encoding for xml file " + file.getAbsolutePath(), e);
}
}
if (!myUrlToEncoding.isEmpty()) {
File current = file;
@@ -71,6 +91,11 @@ public class JpsEncodingProjectConfigurationImpl extends JpsElementBase<JpsEncod
return JpsEncodingConfigurationService.getInstance().getGlobalEncoding(model.getGlobal());
}
private static boolean isXmlFile(File file) {
String fileName = file.getName();
return SystemInfo.isFileSystemCaseSensitive ? fileName.endsWith(XML_NAME_SUFFIX) : StringUtil.endsWithIgnoreCase(fileName, XML_NAME_SUFFIX);
}
@NotNull
@Override
public Map<String, String> getUrlToEncoding() {
jps/model-serialization/testData/fileEncoding/dir/with-encoding.xml
+2
View File
@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="UTF-8" ?>
<test/>
jps/model-serialization/testData/fileEncoding/dir/without-encoding.xml
+1
View File
@@ -0,0 +1 @@
<test/>
jps/model-serialization/testData/fileEncoding/fileEncoding.ipr
+15
View File
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" useUTFGuessing="true" native2AsciiForPropertiesFiles="true" defaultCharsetForPropertiesFiles="UTF-8">
<file url="file://$PROJECT_DIR$/dir" charset="windows-1251" />
<file url="PROJECT" charset="UTF-8" />
</component>
<component name="ProjectModuleManager">
<modules>
</modules>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" assert-keyword="true" jdk-15="true" project-jdk-name="1.7" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
jps/model-serialization/testSrc/org/jetbrains/jps/model/JpsEncodingConfigurationServiceTest.java
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jps.model;
import org.jetbrains.jps.model.serialization.JpsSerializationTestCase;
import java.io.File;
/**
* @author nik
*/
public class JpsEncodingConfigurationServiceTest extends JpsSerializationTestCase {
public void test() {
loadProject("/jps/model-serialization/testData/fileEncoding/fileEncoding.ipr");
JpsEncodingProjectConfiguration configuration = JpsEncodingConfigurationService.getInstance().getEncodingConfiguration(myProject);
assertNotNull(configuration);
assertEncoding("windows-1251", "dir/a.txt", configuration);
assertEncoding("UTF-8", "dir/with-encoding.xml", configuration);
assertEncoding("windows-1251", "dir/without-encoding.xml", configuration);
assertEncoding("windows-1251", "dir/non-existent.xml", configuration);
}
private void assertEncoding(final String encoding, final String path, JpsEncodingProjectConfiguration configuration) {
assertEquals(encoding, configuration.getEncoding(new File(getAbsolutePath(path))));
}
}
platform/core-api/src/com/intellij/openapi/vfs/CharsetToolkit.java
+5 -9
View File
@@ -17,6 +17,7 @@ package com.intellij.openapi.vfs;
import com.intellij.openapi.vfs.encoding.EncodingRegistry;
import com.intellij.util.ArrayUtil;
import com.intellij.util.text.CharsetUtil;
import gnu.trove.THashMap;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
@@ -74,7 +75,7 @@ import java.util.Map;
* @author Guillaume LAFORGE
*/
public class CharsetToolkit {
@NonNls public static final String UTF8 = "UTF-8";
@NonNls public static final String UTF8 = CharsetUtil.UTF8;
public static final Charset UTF8_CHARSET = Charset.forName(UTF8);
public static final Charset UTF_16LE_CHARSET = Charset.forName("UTF-16LE");
public static final Charset UTF_16BE_CHARSET = Charset.forName("UTF-16BE");
@@ -86,7 +87,7 @@ public class CharsetToolkit {
private final Charset defaultCharset;
private boolean enforce8Bit = false;
public static final byte[] UTF8_BOM = {0xffffffef, 0xffffffbb, 0xffffffbf, };
public static final byte[] UTF8_BOM = CharsetUtil.UTF8_BOM;
public static final byte[] UTF16LE_BOM = {-1, -2, };
public static final byte[] UTF16BE_BOM = {-2, -1, };
public static final byte[] UTF32BE_BOM = {0, 0, -2, -1, };
@@ -441,7 +442,7 @@ public class CharsetToolkit {
* @return true if the buffer has a BOM for UTF8.
*/
public static boolean hasUTF8Bom(@NotNull byte[] bom) {
return ArrayUtil.startsWith(bom, UTF8_BOM);
return CharsetUtil.hasUTF8Bom(bom);
}
/**
@@ -486,12 +487,7 @@ public class CharsetToolkit {
@NotNull
public static byte[] getUtf8Bytes(@NotNull String s) {
try {
return s.getBytes(UTF8);
}
catch (UnsupportedEncodingException e) {
throw new RuntimeException("UTF-8 must be supported", e);
}
return CharsetUtil.getUtf8Bytes(s);
}
public static int getBOMLength(@NotNull byte[] content, Charset charset) {
platform/util/src/com/intellij/util/text/CharsetUtil.java
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.text;
import com.intellij.util.ArrayUtil;
import org.jetbrains.annotations.NonNls;
import java.io.UnsupportedEncodingException;
/**
* @author nik
*/
public class CharsetUtil {
public static final byte[] UTF8_BOM = {0xffffffef, 0xffffffbb, 0xffffffbf};
@NonNls public static final String UTF8 = "UTF-8";
public static boolean hasUTF8Bom(byte[] bom) {
return ArrayUtil.startsWith(bom, UTF8_BOM);
}
public static byte[] getUtf8Bytes(String s) {
try {
return s.getBytes(UTF8);
}
catch (UnsupportedEncodingException e) {
throw new RuntimeException("UTF-8 must be supported", e);
}
}
}
platform/util/src/com/intellij/util/text/XmlCharsetDetector.java
+118
View File
@@ -0,0 +1,118 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.text;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.ArrayUtil;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* @author nik
*/
public class XmlCharsetDetector {
@NonNls private static final String XML_PROLOG_START = "<?xml";
@NonNls private static final byte[] XML_PROLOG_START_BYTES = CharsetUtil.getUtf8Bytes(XML_PROLOG_START);
@NonNls private static final String ENCODING = "encoding";
@NonNls private static final byte[] ENCODING_BYTES = CharsetUtil.getUtf8Bytes(ENCODING);
@NonNls private static final String XML_PROLOG_END = "?>";
@NonNls private static final byte[] XML_PROLOG_END_BYTES = CharsetUtil.getUtf8Bytes(XML_PROLOG_END);
@Nullable
public static String extractXmlEncodingFromProlog(final byte[] bytes) {
int index = 0;
if (CharsetUtil.hasUTF8Bom(bytes)) {
index = CharsetUtil.UTF8_BOM.length;
}
index = skipWhiteSpace(index, bytes);
if (!ArrayUtil.startsWith(bytes, index, XML_PROLOG_START_BYTES)) return null;
index += XML_PROLOG_START_BYTES.length;
while (index < bytes.length) {
index = skipWhiteSpace(index, bytes);
if (ArrayUtil.startsWith(bytes, index, XML_PROLOG_END_BYTES)) return null;
if (ArrayUtil.startsWith(bytes, index, ENCODING_BYTES)) {
index += ENCODING_BYTES.length;
index = skipWhiteSpace(index, bytes);
if (index >= bytes.length || bytes[index] != '=') continue;
index++;
index = skipWhiteSpace(index, bytes);
if (index >= bytes.length || bytes[index] != '\'' && bytes[index] != '\"') continue;
byte quote = bytes[index];
index++;
StringBuilder encoding = new StringBuilder();
while (index < bytes.length) {
if (bytes[index] == quote) return encoding.toString();
encoding.append((char)bytes[index++]);
}
}
index++;
}
return null;
}
@Nullable
public static String extractXmlEncodingFromProlog(@NotNull String text) {
int index = 0;
index = skipWhiteSpace(index, text);
if (!StringUtil.startsWith(text, index, XML_PROLOG_START)) return null;
index += XML_PROLOG_START.length();
while (index < text.length()) {
index = skipWhiteSpace(index, text);
if (StringUtil.startsWith(text, index, XML_PROLOG_END)) return null;
if (StringUtil.startsWith(text, index, ENCODING)) {
index += ENCODING.length();
index = skipWhiteSpace(index, text);
if (index >= text.length() || text.charAt(index) != '=') continue;
index++;
index = skipWhiteSpace(index, text);
if (index >= text.length()) continue;
char quote = text.charAt(index);
if (quote != '\'' && quote != '\"') continue;
index++;
StringBuilder encoding = new StringBuilder();
while (index < text.length()) {
char c = text.charAt(index);
if (c == quote) return encoding.toString();
encoding.append(c);
index++;
}
}
index++;
}
return null;
}
private static int skipWhiteSpace(int start, @NotNull byte[] bytes) {
while (start < bytes.length) {
char c = (char)bytes[start];
if (!Character.isWhitespace(c)) break;
start++;
}
return start;
}
private static int skipWhiteSpace(int start, @NotNull String text) {
while (start < text.length()) {
char c = text.charAt(start);
if (!Character.isWhitespace(c)) break;
start++;
}
return start;
}
}
xml/impl/src/com/intellij/xml/util/XmlUtil.java
+3 -91
View File
@@ -60,6 +60,7 @@ import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.psi.xml.*;
import com.intellij.util.*;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.text.XmlCharsetDetector;
import com.intellij.xml.*;
import com.intellij.xml.impl.schema.ComplexTypeDescriptor;
import com.intellij.xml.impl.schema.TypeDescriptor;
@@ -1518,103 +1519,14 @@ public class XmlUtil {
return StringUtil.escapeXml(text);
}
@NonNls private static final String XML_PROLOG_START = "<?xml";
@NonNls private static final byte[] XML_PROLOG_START_BYTES = CharsetToolkit.getUtf8Bytes(XML_PROLOG_START);
@NonNls private static final String ENCODING = "encoding";
@NonNls private static final byte[] ENCODING_BYTES = CharsetToolkit.getUtf8Bytes(ENCODING);
@NonNls private static final String XML_PROLOG_END = "?>";
@NonNls private static final byte[] XML_PROLOG_END_BYTES = CharsetToolkit.getUtf8Bytes(XML_PROLOG_END);
@Nullable
public static String extractXmlEncodingFromProlog(final byte[] content) {
return detect(content);
}
@Nullable
private static String detect(final byte[] bytes) {
int index = 0;
if (CharsetToolkit.hasUTF8Bom(bytes)) {
index = CharsetToolkit.UTF8_BOM.length;
}
index = skipWhiteSpace(index, bytes);
if (!ArrayUtil.startsWith(bytes, index, XML_PROLOG_START_BYTES)) return null;
index += XML_PROLOG_START_BYTES.length;
while (index < bytes.length) {
index = skipWhiteSpace(index, bytes);
if (ArrayUtil.startsWith(bytes, index, XML_PROLOG_END_BYTES)) return null;
if (ArrayUtil.startsWith(bytes, index, ENCODING_BYTES)) {
index += ENCODING_BYTES.length;
index = skipWhiteSpace(index, bytes);
if (index >= bytes.length || bytes[index] != '=') continue;
index++;
index = skipWhiteSpace(index, bytes);
if (index >= bytes.length || bytes[index] != '\'' && bytes[index] != '\"') continue;
byte quote = bytes[index];
index++;
StringBuilder encoding = new StringBuilder();
while (index < bytes.length) {
if (bytes[index] == quote) return encoding.toString();
encoding.append((char)bytes[index++]);
}
}
index++;
}
return null;
}
@Nullable
private static String detect(@NotNull String text) {
int index = 0;
index = skipWhiteSpace(index, text);
if (!StringUtil.startsWith(text, index, XML_PROLOG_START)) return null;
index += XML_PROLOG_START.length();
while (index < text.length()) {
index = skipWhiteSpace(index, text);
if (StringUtil.startsWith(text, index, XML_PROLOG_END)) return null;
if (StringUtil.startsWith(text, index, ENCODING)) {
index += ENCODING.length();
index = skipWhiteSpace(index, text);
if (index >= text.length() || text.charAt(index) != '=') continue;
index++;
index = skipWhiteSpace(index, text);
if (index >= text.length()) continue;
char quote = text.charAt(index);
if (quote != '\'' && quote != '\"') continue;
index++;
StringBuilder encoding = new StringBuilder();
while (index < text.length()) {
char c = text.charAt(index);
if (c == quote) return encoding.toString();
encoding.append(c);
index++;
}
}
index++;
}
return null;
}
private static int skipWhiteSpace(int start, @NotNull byte[] bytes) {
while (start < bytes.length) {
char c = (char)bytes[start];
if (!Character.isWhitespace(c)) break;
start++;
}
return start;
}
private static int skipWhiteSpace(int start, @NotNull String text) {
while (start < text.length()) {
char c = text.charAt(start);
if (!Character.isWhitespace(c)) break;
start++;
}
return start;
return XmlCharsetDetector.extractXmlEncodingFromProlog(content);
}
@Nullable
public static String extractXmlEncodingFromProlog(String text) {
return detect(text);
return XmlCharsetDetector.extractXmlEncodingFromProlog(text);
}
public static void registerXmlAttributeValueReferenceProvider(PsiReferenceRegistrar registrar,