From 837245dcb3afd304c65b43e54785ce4791c54bfd Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 18 Dec 2012 11:29:09 +0400 Subject: [PATCH] external build: honor encoding specified in xml files (IDEA-97558) --- .../JpsEncodingProjectConfigurationImpl.java | 25 ++++ .../fileEncoding/dir/with-encoding.xml | 2 + .../fileEncoding/dir/without-encoding.xml | 1 + .../testData/fileEncoding/fileEncoding.ipr | 15 +++ .../JpsEncodingConfigurationServiceTest.java | 39 ++++++ .../intellij/openapi/vfs/CharsetToolkit.java | 14 +-- .../com/intellij/util/text/CharsetUtil.java | 42 +++++++ .../util/text/XmlCharsetDetector.java | 118 ++++++++++++++++++ .../src/com/intellij/xml/util/XmlUtil.java | 94 +------------- 9 files changed, 250 insertions(+), 100 deletions(-) create mode 100644 jps/model-serialization/testData/fileEncoding/dir/with-encoding.xml create mode 100644 jps/model-serialization/testData/fileEncoding/dir/without-encoding.xml create mode 100644 jps/model-serialization/testData/fileEncoding/fileEncoding.ipr create mode 100644 jps/model-serialization/testSrc/org/jetbrains/jps/model/JpsEncodingConfigurationServiceTest.java create mode 100644 platform/util/src/com/intellij/util/text/CharsetUtil.java create mode 100644 platform/util/src/com/intellij/util/text/XmlCharsetDetector.java diff --git a/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsEncodingProjectConfigurationImpl.java b/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsEncodingProjectConfigurationImpl.java index fe832369c14c..1b7ce68c8277 100644 --- a/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsEncodingProjectConfigurationImpl.java +++ b/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsEncodingProjectConfigurationImpl.java @@ -15,7 +15,12 @@ */ package org.jetbrains.jps.model.impl; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.SystemInfo; +import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.io.FileUtilRt; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.text.XmlCharsetDetector; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.model.JpsElementChildRole; @@ -27,6 +32,7 @@ import org.jetbrains.jps.model.ex.JpsElementChildRoleBase; import org.jetbrains.jps.util.JpsPathUtil; import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -36,7 +42,9 @@ import java.util.Map; */ public class JpsEncodingProjectConfigurationImpl extends JpsElementBase implements JpsEncodingProjectConfiguration { + private static final Logger LOG = Logger.getInstance(JpsEncodingProjectConfigurationImpl.class); public static final JpsElementChildRole ROLE = JpsElementChildRoleBase.create("encoding configuration"); + private static final String XML_NAME_SUFFIX = ".xml"; private final Map myUrlToEncoding = new HashMap(); private final String myProjectEncoding; @@ -48,6 +56,18 @@ public class JpsEncodingProjectConfigurationImpl extends JpsElementBase getUrlToEncoding() { diff --git a/jps/model-serialization/testData/fileEncoding/dir/with-encoding.xml b/jps/model-serialization/testData/fileEncoding/dir/with-encoding.xml new file mode 100644 index 000000000000..3fb0f4d22133 --- /dev/null +++ b/jps/model-serialization/testData/fileEncoding/dir/with-encoding.xml @@ -0,0 +1,2 @@ + + diff --git a/jps/model-serialization/testData/fileEncoding/dir/without-encoding.xml b/jps/model-serialization/testData/fileEncoding/dir/without-encoding.xml new file mode 100644 index 000000000000..52e0a2001de9 --- /dev/null +++ b/jps/model-serialization/testData/fileEncoding/dir/without-encoding.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/jps/model-serialization/testData/fileEncoding/fileEncoding.ipr b/jps/model-serialization/testData/fileEncoding/fileEncoding.ipr new file mode 100644 index 000000000000..6a7bb958fa39 --- /dev/null +++ b/jps/model-serialization/testData/fileEncoding/fileEncoding.ipr @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/jps/model-serialization/testSrc/org/jetbrains/jps/model/JpsEncodingConfigurationServiceTest.java b/jps/model-serialization/testSrc/org/jetbrains/jps/model/JpsEncodingConfigurationServiceTest.java new file mode 100644 index 000000000000..b1f704a9d37b --- /dev/null +++ b/jps/model-serialization/testSrc/org/jetbrains/jps/model/JpsEncodingConfigurationServiceTest.java @@ -0,0 +1,39 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jetbrains.jps.model; + +import org.jetbrains.jps.model.serialization.JpsSerializationTestCase; + +import java.io.File; + +/** + * @author nik + */ +public class JpsEncodingConfigurationServiceTest extends JpsSerializationTestCase { + public void test() { + loadProject("/jps/model-serialization/testData/fileEncoding/fileEncoding.ipr"); + JpsEncodingProjectConfiguration configuration = JpsEncodingConfigurationService.getInstance().getEncodingConfiguration(myProject); + assertNotNull(configuration); + assertEncoding("windows-1251", "dir/a.txt", configuration); + assertEncoding("UTF-8", "dir/with-encoding.xml", configuration); + assertEncoding("windows-1251", "dir/without-encoding.xml", configuration); + assertEncoding("windows-1251", "dir/non-existent.xml", configuration); + } + + private void assertEncoding(final String encoding, final String path, JpsEncodingProjectConfiguration configuration) { + assertEquals(encoding, configuration.getEncoding(new File(getAbsolutePath(path)))); + } +} diff --git a/platform/core-api/src/com/intellij/openapi/vfs/CharsetToolkit.java b/platform/core-api/src/com/intellij/openapi/vfs/CharsetToolkit.java index bbb4e3ed2eca..ce0aae9c5177 100644 --- a/platform/core-api/src/com/intellij/openapi/vfs/CharsetToolkit.java +++ b/platform/core-api/src/com/intellij/openapi/vfs/CharsetToolkit.java @@ -17,6 +17,7 @@ package com.intellij.openapi.vfs; import com.intellij.openapi.vfs.encoding.EncodingRegistry; import com.intellij.util.ArrayUtil; +import com.intellij.util.text.CharsetUtil; import gnu.trove.THashMap; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; @@ -74,7 +75,7 @@ import java.util.Map; * @author Guillaume LAFORGE */ public class CharsetToolkit { - @NonNls public static final String UTF8 = "UTF-8"; + @NonNls public static final String UTF8 = CharsetUtil.UTF8; public static final Charset UTF8_CHARSET = Charset.forName(UTF8); public static final Charset UTF_16LE_CHARSET = Charset.forName("UTF-16LE"); public static final Charset UTF_16BE_CHARSET = Charset.forName("UTF-16BE"); @@ -86,7 +87,7 @@ public class CharsetToolkit { private final Charset defaultCharset; private boolean enforce8Bit = false; - public static final byte[] UTF8_BOM = {0xffffffef, 0xffffffbb, 0xffffffbf, }; + public static final byte[] UTF8_BOM = CharsetUtil.UTF8_BOM; public static final byte[] UTF16LE_BOM = {-1, -2, }; public static final byte[] UTF16BE_BOM = {-2, -1, }; public static final byte[] UTF32BE_BOM = {0, 0, -2, -1, }; @@ -441,7 +442,7 @@ public class CharsetToolkit { * @return true if the buffer has a BOM for UTF8. */ public static boolean hasUTF8Bom(@NotNull byte[] bom) { - return ArrayUtil.startsWith(bom, UTF8_BOM); + return CharsetUtil.hasUTF8Bom(bom); } /** @@ -486,12 +487,7 @@ public class CharsetToolkit { @NotNull public static byte[] getUtf8Bytes(@NotNull String s) { - try { - return s.getBytes(UTF8); - } - catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 must be supported", e); - } + return CharsetUtil.getUtf8Bytes(s); } public static int getBOMLength(@NotNull byte[] content, Charset charset) { diff --git a/platform/util/src/com/intellij/util/text/CharsetUtil.java b/platform/util/src/com/intellij/util/text/CharsetUtil.java new file mode 100644 index 000000000000..271e9acbb983 --- /dev/null +++ b/platform/util/src/com/intellij/util/text/CharsetUtil.java @@ -0,0 +1,42 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.text; + +import com.intellij.util.ArrayUtil; +import org.jetbrains.annotations.NonNls; + +import java.io.UnsupportedEncodingException; + +/** + * @author nik + */ +public class CharsetUtil { + public static final byte[] UTF8_BOM = {0xffffffef, 0xffffffbb, 0xffffffbf}; + @NonNls public static final String UTF8 = "UTF-8"; + + public static boolean hasUTF8Bom(byte[] bom) { + return ArrayUtil.startsWith(bom, UTF8_BOM); + } + + public static byte[] getUtf8Bytes(String s) { + try { + return s.getBytes(UTF8); + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException("UTF-8 must be supported", e); + } + } +} diff --git a/platform/util/src/com/intellij/util/text/XmlCharsetDetector.java b/platform/util/src/com/intellij/util/text/XmlCharsetDetector.java new file mode 100644 index 000000000000..372eb755032b --- /dev/null +++ b/platform/util/src/com/intellij/util/text/XmlCharsetDetector.java @@ -0,0 +1,118 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.text; + +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.ArrayUtil; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * @author nik + */ +public class XmlCharsetDetector { + @NonNls private static final String XML_PROLOG_START = ""; + @NonNls private static final byte[] XML_PROLOG_END_BYTES = CharsetUtil.getUtf8Bytes(XML_PROLOG_END); + + @Nullable + public static String extractXmlEncodingFromProlog(final byte[] bytes) { + int index = 0; + if (CharsetUtil.hasUTF8Bom(bytes)) { + index = CharsetUtil.UTF8_BOM.length; + } + + index = skipWhiteSpace(index, bytes); + if (!ArrayUtil.startsWith(bytes, index, XML_PROLOG_START_BYTES)) return null; + index += XML_PROLOG_START_BYTES.length; + while (index < bytes.length) { + index = skipWhiteSpace(index, bytes); + if (ArrayUtil.startsWith(bytes, index, XML_PROLOG_END_BYTES)) return null; + if (ArrayUtil.startsWith(bytes, index, ENCODING_BYTES)) { + index += ENCODING_BYTES.length; + index = skipWhiteSpace(index, bytes); + if (index >= bytes.length || bytes[index] != '=') continue; + index++; + index = skipWhiteSpace(index, bytes); + if (index >= bytes.length || bytes[index] != '\'' && bytes[index] != '\"') continue; + byte quote = bytes[index]; + index++; + StringBuilder encoding = new StringBuilder(); + while (index < bytes.length) { + if (bytes[index] == quote) return encoding.toString(); + encoding.append((char)bytes[index++]); + } + } + index++; + } + return null; + } + + @Nullable + public static String extractXmlEncodingFromProlog(@NotNull String text) { + int index = 0; + + index = skipWhiteSpace(index, text); + if (!StringUtil.startsWith(text, index, XML_PROLOG_START)) return null; + index += XML_PROLOG_START.length(); + while (index < text.length()) { + index = skipWhiteSpace(index, text); + if (StringUtil.startsWith(text, index, XML_PROLOG_END)) return null; + if (StringUtil.startsWith(text, index, ENCODING)) { + index += ENCODING.length(); + index = skipWhiteSpace(index, text); + if (index >= text.length() || text.charAt(index) != '=') continue; + index++; + index = skipWhiteSpace(index, text); + if (index >= text.length()) continue; + char quote = text.charAt(index); + if (quote != '\'' && quote != '\"') continue; + index++; + StringBuilder encoding = new StringBuilder(); + while (index < text.length()) { + char c = text.charAt(index); + if (c == quote) return encoding.toString(); + encoding.append(c); + index++; + } + } + index++; + } + return null; + } + + private static int skipWhiteSpace(int start, @NotNull byte[] bytes) { + while (start < bytes.length) { + char c = (char)bytes[start]; + if (!Character.isWhitespace(c)) break; + start++; + } + return start; + } + + private static int skipWhiteSpace(int start, @NotNull String text) { + while (start < text.length()) { + char c = text.charAt(start); + if (!Character.isWhitespace(c)) break; + start++; + } + return start; + } +} diff --git a/xml/impl/src/com/intellij/xml/util/XmlUtil.java b/xml/impl/src/com/intellij/xml/util/XmlUtil.java index fd5256a45aae..fec8120c2442 100644 --- a/xml/impl/src/com/intellij/xml/util/XmlUtil.java +++ b/xml/impl/src/com/intellij/xml/util/XmlUtil.java @@ -60,6 +60,7 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.psi.xml.*; import com.intellij.util.*; import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.text.XmlCharsetDetector; import com.intellij.xml.*; import com.intellij.xml.impl.schema.ComplexTypeDescriptor; import com.intellij.xml.impl.schema.TypeDescriptor; @@ -1518,103 +1519,14 @@ public class XmlUtil { return StringUtil.escapeXml(text); } - @NonNls private static final String XML_PROLOG_START = ""; - @NonNls private static final byte[] XML_PROLOG_END_BYTES = CharsetToolkit.getUtf8Bytes(XML_PROLOG_END); - @Nullable public static String extractXmlEncodingFromProlog(final byte[] content) { - return detect(content); - } - - @Nullable - private static String detect(final byte[] bytes) { - int index = 0; - if (CharsetToolkit.hasUTF8Bom(bytes)) { - index = CharsetToolkit.UTF8_BOM.length; - } - - index = skipWhiteSpace(index, bytes); - if (!ArrayUtil.startsWith(bytes, index, XML_PROLOG_START_BYTES)) return null; - index += XML_PROLOG_START_BYTES.length; - while (index < bytes.length) { - index = skipWhiteSpace(index, bytes); - if (ArrayUtil.startsWith(bytes, index, XML_PROLOG_END_BYTES)) return null; - if (ArrayUtil.startsWith(bytes, index, ENCODING_BYTES)) { - index += ENCODING_BYTES.length; - index = skipWhiteSpace(index, bytes); - if (index >= bytes.length || bytes[index] != '=') continue; - index++; - index = skipWhiteSpace(index, bytes); - if (index >= bytes.length || bytes[index] != '\'' && bytes[index] != '\"') continue; - byte quote = bytes[index]; - index++; - StringBuilder encoding = new StringBuilder(); - while (index < bytes.length) { - if (bytes[index] == quote) return encoding.toString(); - encoding.append((char)bytes[index++]); - } - } - index++; - } - return null; - } - @Nullable - private static String detect(@NotNull String text) { - int index = 0; - - index = skipWhiteSpace(index, text); - if (!StringUtil.startsWith(text, index, XML_PROLOG_START)) return null; - index += XML_PROLOG_START.length(); - while (index < text.length()) { - index = skipWhiteSpace(index, text); - if (StringUtil.startsWith(text, index, XML_PROLOG_END)) return null; - if (StringUtil.startsWith(text, index, ENCODING)) { - index += ENCODING.length(); - index = skipWhiteSpace(index, text); - if (index >= text.length() || text.charAt(index) != '=') continue; - index++; - index = skipWhiteSpace(index, text); - if (index >= text.length()) continue; - char quote = text.charAt(index); - if (quote != '\'' && quote != '\"') continue; - index++; - StringBuilder encoding = new StringBuilder(); - while (index < text.length()) { - char c = text.charAt(index); - if (c == quote) return encoding.toString(); - encoding.append(c); - index++; - } - } - index++; - } - return null; - } - - private static int skipWhiteSpace(int start, @NotNull byte[] bytes) { - while (start < bytes.length) { - char c = (char)bytes[start]; - if (!Character.isWhitespace(c)) break; - start++; - } - return start; - } - private static int skipWhiteSpace(int start, @NotNull String text) { - while (start < text.length()) { - char c = text.charAt(start); - if (!Character.isWhitespace(c)) break; - start++; - } - return start; + return XmlCharsetDetector.extractXmlEncodingFromProlog(content); } @Nullable public static String extractXmlEncodingFromProlog(String text) { - return detect(text); + return XmlCharsetDetector.extractXmlEncodingFromProlog(text); } public static void registerXmlAttributeValueReferenceProvider(PsiReferenceRegistrar registrar,