MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

FileEncodingTest moved to community

Browse Source
This commit is contained in:
Alexey Kudravtsev
2017-02-16 10:24:52 +03:00
parent 8e812ac6c6
commit 2884a40691
16 changed files with 1265 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/platform-tests/testData/vfs/encoding/.gitattributes
View File
platform/platform-tests/testData/vfs/encoding/UTF16BOM.txt
+1
View File
@@ -0,0 +1 @@
При
platform/platform-tests/testData/vfs/encoding/UTF16LE_NOBOM.txt
+1
View File
@@ -0,0 +1 @@
@825B
platform/platform-tests/testData/vfs/encoding/defaultHtml.html
+1
View File
@@ -0,0 +1 @@
<html></html>
platform/platform-tests/testData/vfs/encoding/newEncoding/.idea/encodings.xml
Generated
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/src/xxx.txt" charset="US-ASCII" />
<file url="PROJECT" charset="UTF-8" />
</component>
</project>
platform/platform-tests/testData/vfs/encoding/newEncoding/.idea/misc.xml
Generated
+23
View File
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MavenImportPreferences">
<option name="generalSettings">
<MavenGeneralSettings>
<option name="mavenHome" value="Bundled (Maven 3)" />
</MavenGeneralSettings>
</option>
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="false">
<OptionsSetting value="true" id="Add" />
<OptionsSetting value="true" id="Remove" />
<OptionsSetting value="true" id="Checkout" />
<OptionsSetting value="true" id="Update" />
<OptionsSetting value="true" id="Status" />
<OptionsSetting value="true" id="Edit" />
<ConfirmationsSetting value="0" id="Add" />
<ConfirmationsSetting value="0" id="Remove" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
platform/platform-tests/testData/vfs/encoding/newEncoding/.idea/modules.xml
Generated
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/newEncoding.iml" filepath="$PROJECT_DIR$/newEncoding.iml" />
</modules>
</component>
</project>
platform/platform-tests/testData/vfs/encoding/newEncoding/.idea/workspace.xml
Generated
+299
View File
@@ -0,0 +1,299 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="7e6de6cf-3eea-4d47-a0e8-461e926624c9" name="Default" comment="" />
<ignored path="newEncoding.iws" />
<ignored path=".idea/workspace.xml" />
<ignored path="$PROJECT_DIR$/out/" />
<ignored path=".idea/dataSources.local.xml" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="CreatePatchCommitExecutor">
<option name="PATCH_PATH" value="" />
</component>
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="FavoritesManager">
<favorites_list name="newEncoding" />
</component>
<component name="FileEditorManager">
<leaf>
<file leaf-file-name="xxx.txt" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/xxx.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="16" selection-start-line="0" selection-start-column="16" selection-end-line="0" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="GradleLocalSettings">
<option name="externalProjectsViewState">
<projects_view />
</option>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/xxx.txt" />
</list>
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="PhpServers">
<servers />
</component>
<component name="PhpWorkspaceProjectConfiguration" backward_compatibility_performed="true" />
<component name="ProjectFrameBounds">
<option name="x" value="1276" />
<option name="y" value="-4" />
<option name="width" value="1288" />
<option name="height" value="1032" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="false">
<OptionsSetting value="true" id="Add" />
<OptionsSetting value="true" id="Remove" />
<OptionsSetting value="true" id="Checkout" />
<OptionsSetting value="true" id="Update" />
<OptionsSetting value="true" id="Status" />
<OptionsSetting value="true" id="Edit" />
<ConfirmationsSetting value="0" id="Add" />
<ConfirmationsSetting value="0" id="Remove" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scratches" />
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="newEncoding" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="newEncoding" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="newEncoding" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="newEncoding" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="newEncoding" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane>
</pane>
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="settings.editor.selected.configurable" value="preferences.pluginManager" />
<property name="settings.editor.splitter.proportion" value="0.2" />
<property name="aspect.path.notification.shown" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
</component>
<component name="RunManager">
<configuration default="true" type="#org.jetbrains.idea.devkit.run.PluginConfigurationType" factoryName="Plugin">
<module name="" />
<option name="VM_PARAMETERS" value="-Xmx512m -Xms256m -XX:MaxPermSize=250m -ea" />
<option name="PROGRAM_PARAMETERS" />
<method />
</configuration>
<configuration default="true" type="Applet" factoryName="Applet">
<option name="HTML_USED" value="false" />
<option name="WIDTH" value="400" />
<option name="HEIGHT" value="300" />
<option name="POLICY_FILE" value="$APPLICATION_HOME_DIR$/bin/appletviewer.policy" />
<module />
<method />
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" />
<option name="VM_PARAMETERS" />
<option name="PROGRAM_PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="ENABLE_SWING_INSPECTOR" value="false" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<module name="" />
<envs />
<method />
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$MODULE_DIR$" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<envs />
<patterns />
<method />
</configuration>
<configuration default="true" type="Remote" factoryName="Remote">
<option name="USE_SOCKET_TRANSPORT" value="true" />
<option name="SERVER_MODE" value="false" />
<option name="SHMEM_ADDRESS" value="javadebug" />
<option name="HOST" value="localhost" />
<option name="PORT" value="5005" />
<method />
</configuration>
<configuration default="true" type="TestNG" factoryName="TestNG">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="SUITE_NAME" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="GROUP_NAME" />
<option name="TEST_OBJECT" value="CLASS" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$MODULE_DIR$" />
<option name="OUTPUT_DIRECTORY" />
<option name="ANNOTATION_TYPE" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<option name="USE_DEFAULT_REPORTERS" value="false" />
<option name="PROPERTIES_FILE" />
<envs />
<properties />
<listeners />
<method />
</configuration>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="7e6de6cf-3eea-4d47-a0e8-461e926624c9" name="Default" comment="" />
<created>1464094338548</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1464094338548</updated>
<workItem from="1464094342515" duration="44000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="44000" />
</component>
<component name="ToolWindowManager">
<frame x="1276" y="-4" width="1288" height="1032" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24959612" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Profiler" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="Vcs.Log.UiProperties">
<option name="RECENTLY_FILTERED_USER_GROUPS">
<collection />
</option>
<option name="RECENTLY_FILTERED_BRANCH_GROUPS">
<collection />
</option>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/src/xxx.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="16" selection-start-line="0" selection-start-column="16" selection-end-line="0" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
</component>
</project>
platform/platform-tests/testData/vfs/encoding/newEncoding/newEncoding.iml
+11
View File
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
platform/platform-tests/testData/vfs/encoding/newEncoding/src/xxx.txt
+1
View File
@@ -0,0 +1 @@
xxxxxxxxxxxxxxxx
platform/platform-tests/testData/vfs/encoding/xDefault.xml
+3
View File
@@ -0,0 +1,3 @@
<?xml version="1.0"?>
<web-app>
</web-app>
platform/platform-tests/testData/vfs/encoding/xIbm866.xml
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="IBM866"?>
<web-app>
<!--¯ ¯ -->
</web-app>
platform/platform-tests/testData/vfs/encoding/xNotepadUtf8.xml
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<web-app>
<!--папа-->
</web-app>
platform/platform-tests/testData/vfs/encoding/xTrickyProlog.xml
+4
View File
@@ -0,0 +1,4 @@
<?xml encoding='Big5' version='1.0' ?>
<web-app>
</web-app>
platform/platform-tests/testData/vfs/encoding/xWin1251.xml
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="windows-1251"?>
<web-app>
<!--ïàïà-->
</web-app>
platform/platform-tests/testSrc/com/intellij/openapi/vfs/encoding/FileEncodingTest.java
+894
View File
@@ -0,0 +1,894 @@
/*
* Copyright 2000-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.openapi.vfs.encoding;
import com.intellij.ide.impl.ProjectUtil;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.application.Result;
import com.intellij.openapi.application.WriteAction;
import com.intellij.openapi.application.ex.PathManagerEx;
import com.intellij.openapi.command.CommandProcessor;
import com.intellij.openapi.command.WriteCommandAction;
import com.intellij.openapi.command.undo.UndoManager;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.editor.event.DocumentAdapter;
import com.intellij.openapi.editor.event.DocumentEvent;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
import com.intellij.openapi.fileTypes.PlainTextFileType;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ex.ProjectManagerEx;
import com.intellij.openapi.roots.ModuleRootManager;
import com.intellij.openapi.ui.Messages;
import com.intellij.openapi.ui.TestDialog;
import com.intellij.openapi.util.Disposer;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.openapi.util.io.IoTestUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.*;
import com.intellij.psi.PsiDirectory;
import com.intellij.psi.PsiFile;
import com.intellij.psi.PsiManager;
import com.intellij.refactoring.copy.CopyFilesOrDirectoriesHandler;
import com.intellij.testFramework.PlatformTestCase;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.testFramework.PsiTestUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.text.XmlCharsetDetector;
import com.intellij.util.ui.UIUtil;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
import static org.junit.Assert.assertArrayEquals;
@SuppressWarnings("HardCodedStringLiteral")
public class FileEncodingTest extends PlatformTestCase implements TestDialog {
private static final Charset US_ASCII = Charset.forName("US-ASCII");
private static final Charset WINDOWS_1251 = Charset.forName("windows-1251");
private static final Charset WINDOWS_1252 = Charset.forName("windows-1252");
private static final String UTF8_XML_PROLOG = prolog(CharsetToolkit.UTF8_CHARSET);
private static final byte[] NO_BOM = ArrayUtil.EMPTY_BYTE_ARRAY;
private static final String XML_TEST_BODY = "<web-app>\n" + "<!--\u043f\u0430\u043f\u0430-->\n" + "</web-app>";
private static final String THREE_RUSSIAN_LETTERS = "\u0416\u041e\u041f";
private TestDialog myOldTestDialogValue;
private static String prolog(Charset charset) {
return "<?xml version=\"1.0\" encoding=\"" + charset.name() + "\"?>\n";
}
@Override
protected void setUp() throws Exception {
super.setUp();
myOldTestDialogValue = Messages.setTestDialog(this);
}
@Override
protected void tearDown() throws Exception {
Messages.setTestDialog(myOldTestDialogValue);
super.tearDown();
}
@Override
public int show(String message) {
return 0;
}
private static Document getDocument(VirtualFile file) {
return FileDocumentManager.getInstance().getDocument(file);
}
public void testWin1251() {
VirtualFile vTestRoot = getTestRoot();
VirtualFile xml = vTestRoot.findChild("xWin1251.xml");
String expected = prolog(WINDOWS_1251) + XML_TEST_BODY;
String text = getDocument(xml).getText();
assertEquals(expected, text);
}
public void testXmlProlog() throws IOException {
VirtualFile vTestRoot = getTestRoot();
VirtualFile xml = vTestRoot.findChild("xNotepadUtf8.xml");
String expected = UTF8_XML_PROLOG + XML_TEST_BODY;
String text = getDocument(xml).getText();
if (!expected.equals(text)) {
System.out.print("expected = ");
for (int i=0; i<50;i++) {
final char c = expected.charAt(i);
System.out.print(Integer.toHexString((int)c)+", ");
}
System.out.println("");
System.out.print("expected bytes = ");
byte[] expectedBytes = FileUtil.loadFileBytes(new File(xml.getPath()));
for (int i=0; i<50;i++) {
final byte c = expectedBytes[i];
System.out.print(Integer.toHexString((int)c) + ", ");
}
System.out.println("");
System.out.print("text = ");
for (int i=0; i<50;i++) {
final char c = text.charAt(i);
System.out.print(Integer.toHexString((int)c)+", ");
}
System.out.println("");
System.out.print("text bytes = ");
byte[] textBytes = xml.contentsToByteArray();
for (int i=0; i<50;i++) {
final byte c = textBytes[i];
System.out.print(Integer.toHexString((int)c) + ", ");
}
System.out.println("");
String charsetFromProlog = XmlCharsetDetector.extractXmlEncodingFromProlog(xml.contentsToByteArray());
System.out.println("charsetFromProlog = " + charsetFromProlog);
Charset charset = xml.getCharset();
System.out.println("charset = " + charset);
}
assertEquals(expected, text);
}
public void testChangeToUtfProlog() throws IOException {
VirtualFile src = getTestRoot().findChild("xWin1251.xml");
final File dir = createTempDirectory();
final File file = new File(dir, "copy.xml");
FileUtil.copy(new File(src.getPath()), file);
new WriteCommandAction.Simple(getProject()) {
@Override
protected void run() throws Throwable {
VirtualFile xml = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(file);
Document document = getDocument(xml);
setText(document, UTF8_XML_PROLOG + XML_TEST_BODY);
FileDocumentManager.getInstance().saveAllDocuments();
byte[] savedBytes = FileUtil.loadFileBytes(file);
String saved = new String(savedBytes, CharsetToolkit.UTF8).replace("\r\n", "\n");
String expected = (UTF8_XML_PROLOG + XML_TEST_BODY).replace("\r\n", "\n");
assertEquals(expected, saved);
}
}.execute().throwException();
}
public void testDefaultHtml() {
VirtualFile file = getTestRoot().findChild("defaultHtml.html");
assertEquals(EncodingProjectManager.getInstance(getProject()).getDefaultCharset(), file.getCharset());
}
public void testTrickyProlog() {
VirtualFile xml = getTestRoot().findChild("xTrickyProlog.xml");
assertEquals("Big5", xml.getCharset().name());
}
public void testDefaultXml() {
VirtualFile xml = getTestRoot().findChild("xDefault.xml");
assertEquals(CharsetToolkit.UTF8_CHARSET, xml.getCharset());
}
public void testIbm866() {
VirtualFile xml = getTestRoot().findChild("xIbm866.xml");
String expected = prolog(Charset.forName("IBM866")) + XML_TEST_BODY;
String text = getDocument(xml).getText();
assertEquals(expected, text);
}
private static VirtualFile getTestRoot() {
final File testRoot = new File(PathManagerEx.getCommunityHomePath(), "platform/platform-tests/testData/vfs/encoding");
return LocalFileSystem.getInstance().findFileByIoFile(testRoot);
}
public void testUTF16BOM() throws IOException {
VirtualFile vTestRoot = getTestRoot();
VirtualFile file = vTestRoot.findChild(getTestName(false) + ".txt");
File source = new File(file.getPath());
byte[] bytesSource = FileUtil.loadFileBytes(source);
assertTrue(Arrays.toString(bytesSource), CharsetToolkit.hasUTF16LEBom(bytesSource));
assertEquals("[-1, -2, 31, 4, 64, 4, 56, 4]", Arrays.toString(bytesSource));
Document document = getDocument(file);
String text = document.getText();
assertEquals("\u041f\u0440\u0438", text);
File copy = FileUtil.createTempFile("copy", ".txt");
myFilesToDelete.add(copy);
FileUtil.copy(source, copy);
VirtualFile fileCopy = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(copy);
document = getDocument(fileCopy);
//assertTrue(CharsetToolkit.hasUTF16LEBom(fileCopy.getBOM()));
setText(document, "\u04ab\u04cd\u04ef");
FileDocumentManager.getInstance().saveAllDocuments();
byte[] bytes = FileUtil.loadFileBytes(copy);
assertTrue(Arrays.toString(bytes), CharsetToolkit.hasUTF16LEBom(bytes));
assertEquals("[-1, -2, -85, 4, -51, 4, -17, 4]", Arrays.toString(bytes));
}
public void testMetaHttpEquivHtml() throws IOException {
doHtmlTest("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\">",
"<meta http-equiv=\"Content-Type\" content=\"text/html;charset=utf-8\">");
}
public void testMetaCharsetHtml5() throws IOException {
doHtmlTest("<meta charset =\"windows-1252\">",
"<meta charset =\"utf-8\">");
}
private static void doHtmlTest(final String metaWithWindowsEncoding, final String metaWithUtf8Encoding) throws IOException {
File temp = FileUtil.createTempFile("copy", ".html");
setContentOnDisk(temp, NO_BOM,
"<html><head>" + metaWithWindowsEncoding + "</head>" +
THREE_RUSSIAN_LETTERS +
"</html>",
WINDOWS_1252);
myFilesToDelete.add(temp);
VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(temp);
assertEquals(WINDOWS_1252, file.getCharset());
Document document = getDocument(file);
setText(document, "<html><head>" + metaWithUtf8Encoding + "</head>" +
THREE_RUSSIAN_LETTERS +
"</html>");
FileDocumentManager.getInstance().saveAllDocuments();
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
}
public void testHtmlEncodingPreferBOM() throws IOException {
VirtualFile file = createTempFile("html", CharsetToolkit.UTF8_BOM,
"<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\"></head>" +
THREE_RUSSIAN_LETTERS +
"</html>",
WINDOWS_1252);
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF8_BOM, file.getBOM());
}
public void testHtmlEncodingCaseInsensitive() throws IOException {
VirtualFile file = createTempFile("html", NO_BOM, "<html>\n" +
"<head>\n" +
"<meta http-equiv=\"content-type\" content=\"text/html;charset=us-ascii\"> \n" +
"</head>\n" +
"<body>\n" +
"xyz\n" +
"</body>\n" +
"</html>",
WINDOWS_1252);
assertEquals(US_ASCII, file.getCharset());
}
public void testHtmlContentAttributeOrder() throws IOException {
VirtualFile file =
createTempFile("html", NO_BOM, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\"\n" +
"\t\t\"http://www.w3.org/TR/html4/loose.dtd\">\n" +
"<html> <head>\n" +
"\t<meta content=\"text/html;charset=US-ASCII\" http-equiv=\"Content-Type\">\n" +
"</head> </html>",
WINDOWS_1252);
assertEquals(US_ASCII, file.getCharset());
}
private static void setText(final Document document, final String text) {
ApplicationManager.getApplication().runWriteAction(() -> document.setText(text));
}
public void testXHtmlStuff() throws IOException {
String text = "<xxx>\n</xxx>";
VirtualFile file = createTempFile("xhtml", NO_BOM, text, WINDOWS_1252);
Document document = getDocument(file);
setText(document, prolog(WINDOWS_1251) + document.getText());
FileDocumentManager.getInstance().saveAllDocuments();
assertEquals(WINDOWS_1251, file.getCharset());
setText(document, prolog(US_ASCII) + "\n<xxx></xxx>");
FileDocumentManager.getInstance().saveAllDocuments();
assertEquals(US_ASCII, file.getCharset());
text = prolog(WINDOWS_1252) + "\n<xxx>\n</xxx>";
file = createTempFile("xhtml", NO_BOM, text, WINDOWS_1252);
assertEquals(WINDOWS_1252, file.getCharset());
}
public void testSettingEncodingManually() throws IOException {
final StringBuilder text = new StringBuilder(THREE_RUSSIAN_LETTERS);
VirtualFile file = createTempFile("txt", NO_BOM, text.toString(), WINDOWS_1251);
File ioFile = new File(file.getPath());
EncodingManager.getInstance().setEncoding(file, WINDOWS_1251);
final Document document = getDocument(file);
final boolean[] changed = {false};
document.addDocumentListener(new DocumentAdapter() {
@Override
public void documentChanged(final DocumentEvent event) {
changed[0] = true;
}
});
EncodingManager.getInstance().setEncoding(file, CharsetToolkit.UTF8_CHARSET);
//text in editor changed
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
UIUtil.dispatchAllInvocationEvents();
assertTrue(changed[0]);
changed[0] = false;
FileDocumentManager.getInstance().saveAllDocuments();
byte[] bytes = FileUtil.loadFileBytes(ioFile);
//file on disk is still windows
assertTrue(Arrays.equals(text.toString().getBytes(WINDOWS_1251.name()), bytes));
ApplicationManager.getApplication().runWriteAction(() -> CommandProcessor.getInstance().executeCommand(getProject(), () -> {
document.insertString(0, "x");
text.insert(0, "x");
}, null, null));
assertTrue(changed[0]);
changed[0] = false;
EncodingManager.getInstance().setEncoding(file, US_ASCII);
assertEquals(US_ASCII, file.getCharset());
UIUtil.dispatchAllInvocationEvents();
assertTrue(changed[0]); //reloaded again
//after 'save as us' file on disk changed
bytes = FileUtil.loadFileBytes(ioFile);
assertTrue(Arrays.equals(text.toString().getBytes(US_ASCII.name()), bytes));
}
public void testCopyMove() throws IOException {
File root = createTempDirectory();
File dir1 = new File(root, "dir1"); dir1.mkdir();
File dir2 = new File(root, "dir2"); dir2.mkdir();
final VirtualFile vdir1 = getVirtualFile(dir1);
final VirtualFile vdir2 = getVirtualFile(dir2);
assertNotNull(dir1.getPath(), vdir1);
assertNotNull(dir2.getPath(), vdir2);
EncodingManager.getInstance().setEncoding(vdir1, WINDOWS_1251);
EncodingManager.getInstance().setEncoding(vdir2, US_ASCII);
new WriteCommandAction.Simple(getProject()) {
@Override
protected void run() throws Throwable {
VirtualFile xxx = vdir1.createChildData(this, "xxx.txt");
setFileText(xxx, THREE_RUSSIAN_LETTERS);
assertEquals(WINDOWS_1251, xxx.getCharset());
VirtualFile copied = xxx.copy(this, vdir2, "xxx2.txt");
assertEquals(WINDOWS_1251, copied.getCharset());
VirtualFile xus = vdir2.createChildData(this, "xxxus.txt");
setFileText(xus, THREE_RUSSIAN_LETTERS);
assertEquals(US_ASCII, xus.getCharset());
xus.move(this, vdir1);
assertEquals(US_ASCII, xus.getCharset());
}
}.execute().throwException();
}
public void testCopyNested() throws IOException, IncorrectOperationException {
File root = createTempDirectory();
File dir1 = new File(root, "dir1"); dir1.mkdir();
File dir2 = new File(root, "dir2"); dir2.mkdir();
final VirtualFile vdir1 = getVirtualFile(dir1);
final VirtualFile vdir2 = getVirtualFile(dir2);
assertNotNull(dir1.getPath(), vdir1);
assertNotNull(dir2.getPath(), vdir2);
EncodingManager.getInstance().setEncoding(vdir1, WINDOWS_1251);
EncodingManager.getInstance().setEncoding(vdir2, US_ASCII);
new WriteCommandAction.Simple(getProject()) {
@Override
protected void run() throws Throwable {
VirtualFile winf = vdir1.createChildData(this, "xxx.txt");
PsiDirectory psidir1 = PsiManager.getInstance(getProject()).findDirectory(vdir1);
PsiDirectory psidir2 = PsiManager.getInstance(getProject()).findDirectory(vdir2);
CopyFilesOrDirectoriesHandler.copyToDirectory(psidir1, psidir1.getName(), psidir2);
VirtualFile winfCopy = psidir2.getVirtualFile().findFileByRelativePath(psidir1.getName() + "/" + winf.getName());
assertEquals(WINDOWS_1251, winfCopy.getCharset());
VirtualFile dir1Copy = psidir2.getVirtualFile().findChild(psidir1.getName());
assertEquals(WINDOWS_1251, EncodingManager.getInstance().getEncoding(dir1Copy, false));
assertNull(EncodingManager.getInstance().getEncoding(winfCopy, false));
}
}.execute().throwException();
}
public void testBOMPersistsAfterEditing() throws IOException {
VirtualFile file = createTempFile("txt", CharsetToolkit.UTF8_BOM, XML_TEST_BODY, CharsetToolkit.UTF8_CHARSET);
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF8_BOM, file.getBOM());
Document document = getDocument(file);
setText(document, "hren");
FileDocumentManager.getInstance().saveAllDocuments();
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF8_BOM, file.getBOM());
byte[] bytes = FileUtil.loadFileBytes(VfsUtilCore.virtualToIoFile(file));
assertTrue(CharsetToolkit.hasUTF8Bom(bytes));
}
public void testBOMPersistsAfterIndexRescan() throws IOException {
VirtualFile file = createTempFile("txt", CharsetToolkit.UTF8_BOM, XML_TEST_BODY, CharsetToolkit.UTF8_CHARSET);
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF8_BOM, file.getBOM());
Document document = getDocument(file);
String newContent = "hren";
setText(document, newContent);
FileDocumentManager.getInstance().saveAllDocuments();
byte[] bytes = FileUtil.loadFileBytes(VfsUtilCore.virtualToIoFile(file));
Charset charset = LoadTextUtil.detectCharsetAndSetBOM(file, bytes);
assertEquals(CharsetToolkit.UTF8_CHARSET, charset);
assertArrayEquals(CharsetToolkit.UTF8_BOM, file.getBOM());
assertTrue(CharsetToolkit.hasUTF8Bom(bytes));
}
public void testVFileGetInputStreamIsBOMAware() throws IOException {
VirtualFile vTestRoot = getTestRoot();
VirtualFile file = vTestRoot.findChild("UTF16BOM.txt");
String vfsLoad = VfsUtilCore.loadText(file);
assertEquals("\u041f\u0440\u0438", vfsLoad);
CharSequence ltuText = LoadTextUtil.loadText(file);
assertEquals(vfsLoad, ltuText.toString());
assertArrayEquals(CharsetToolkit.UTF16LE_BOM, file.getBOM());
}
public void testSetCharsetAfter() throws IOException {
VirtualFile vTestRoot = getTestRoot();
VirtualFile file = vTestRoot.findChild("UTF16LE_NOBOM.txt");
file.setCharset(CharsetToolkit.UTF_16LE_CHARSET);
file.setBOM(null);
String vfsLoad = VfsUtilCore.loadText(file);
assertEquals("\u041f\u0440\u0438\u0432\u0435\u0442", vfsLoad);
assertNull(file.getBOM());
}
public void testBOMResetAfterChangingUtf16ToUtf8() throws IOException {
VirtualFile file = createTempFile("txt", CharsetToolkit.UTF16BE_BOM, THREE_RUSSIAN_LETTERS,
CharsetToolkit.UTF_16BE_CHARSET);
assertEquals(CharsetToolkit.UTF_16BE_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF16BE_BOM, file.getBOM());
Document document = getDocument(file);
String newContent = "hren";
setText(document, newContent);
FileDocumentManager.getInstance().saveAllDocuments();
assertEquals(CharsetToolkit.UTF_16BE_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF16BE_BOM, file.getBOM());
EncodingUtil.saveIn(document, null, file, CharsetToolkit.UTF8_CHARSET);
byte[] bytes = FileUtil.loadFileBytes(VfsUtilCore.virtualToIoFile(file));
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
assertNull(file.getBOM());
assertFalse(CharsetToolkit.hasUTF8Bom(bytes));
}
public void testConvertNotAvailableForHtml() throws IOException {
VirtualFile file = createTempFile("html", null,
"<html><head><meta content=\"text/html; charset=utf-8\" http-equiv=\"content-type\"></head>" +
"<body>" + THREE_RUSSIAN_LETTERS +
"</body></html>",
CharsetToolkit.UTF8_CHARSET);
Document document = FileDocumentManager.getInstance().getDocument(file);
assertNotNull(document);
FileDocumentManager.getInstance().saveAllDocuments();
String pair = EncodingUtil.checkCanConvert(file);
assertNotNull(pair);
}
public void testConvertReload() throws IOException {
VirtualFile file = createTempFile("txt", CharsetToolkit.UTF16BE_BOM, THREE_RUSSIAN_LETTERS, CharsetToolkit.UTF_16BE_CHARSET);
assertEquals(CharsetToolkit.UTF_16BE_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF16BE_BOM, file.getBOM());
Document document = getDocument(file);
byte[] bytes = file.contentsToByteArray();
String text = document.getText();
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, WINDOWS_1251), EncodingUtil.Magic8.NO_WAY);
Assert.assertSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, US_ASCII), EncodingUtil.Magic8.NO_WAY);
String failReason = EncodingUtil.checkCanReload(file).second;
assertNotNull(failReason);
EncodingUtil.saveIn(document, null, file, WINDOWS_1251);
bytes = file.contentsToByteArray();
assertEquals(WINDOWS_1251, file.getCharset());
assertNull(file.getBOM());
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, CharsetToolkit.UTF_16LE_CHARSET), EncodingUtil.Magic8.NO_WAY);
Assert.assertSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, US_ASCII), EncodingUtil.Magic8.NO_WAY);
failReason = EncodingUtil.checkCanReload(file).second;
assertNull(failReason);
EncodingUtil.saveIn(document, null, file, CharsetToolkit.UTF_16LE_CHARSET);
bytes = file.contentsToByteArray();
assertEquals(CharsetToolkit.UTF_16LE_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF16LE_BOM, file.getBOM());
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, WINDOWS_1251), EncodingUtil.Magic8.NO_WAY);
Assert.assertSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, US_ASCII), EncodingUtil.Magic8.NO_WAY);
failReason = EncodingUtil.checkCanReload(file).second;
assertNotNull(failReason);
text = "xxx";
setText(document, text);
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, WINDOWS_1251), EncodingUtil.Magic8.NO_WAY);
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, US_ASCII), EncodingUtil.Magic8.NO_WAY);
FileDocumentManager.getInstance().saveAllDocuments();
bytes = file.contentsToByteArray();
failReason = EncodingUtil.checkCanReload(file).second;
assertNotNull(failReason);
text = "qqq";
setText(document, text);
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, WINDOWS_1251), EncodingUtil.Magic8.NO_WAY);
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, US_ASCII), EncodingUtil.Magic8.NO_WAY);
EncodingUtil.saveIn(document, null, file, US_ASCII);
bytes = file.contentsToByteArray();
assertEquals(US_ASCII, file.getCharset());
assertNull(file.getBOM());
Assert.assertNotSame(EncodingUtil.isSafeToConvertTo(file, text, bytes, WINDOWS_1251), EncodingUtil.Magic8.NO_WAY);
failReason = EncodingUtil.checkCanReload(file).second;
assertNull(failReason);
}
private static String toHexString(byte[] b, int start, int end) {
final String hexChar = "0123456789abcdef";
StringBuilder hex = new StringBuilder();
StringBuilder ch = new StringBuilder();
for (int i = start; i < end; i++) {
hex.append(hexChar.charAt((b[i] >> 4) & 0x0f));
hex.append(hexChar.charAt(b[i] & 0x0f));
hex.append(" ");
if ((i-start+1) % 5 == 0) hex.append(" ");
ch.append((char)b[i]);
ch.append(" ");
if ((i-start+1) % 5 == 0) ch.append(" ");
}
return hex + "\n" + ch;
}
public void testSetEncodingForDirectoryChangesEncodingsForEvenNotLoadedFiles() throws IOException {
EncodingProjectManager.getInstance(getProject()).setEncoding(null, CharsetToolkit.UTF8_CHARSET);
File fc = FileUtil.createTempDirectory("", "");
VirtualFile root = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(fc);
PsiTestUtil.addContentRoot(getModule(), root);
VirtualFile file = createChildData(root, "win.txt");
setContentOnDisk(new File(file.getPath()), null, THREE_RUSSIAN_LETTERS, WINDOWS_1251);
file.refresh(false, false);
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
Assert.assertNull(file.getBOM());
EncodingProjectManager.getInstance(getProject()).setEncoding(root, WINDOWS_1251);
UIUtil.dispatchAllInvocationEvents();
assertEquals(WINDOWS_1251, file.getCharset());
Assert.assertNull(file.getBOM());
}
public void testExternalChangeClearsAutoDetectedFromBytesFlag() throws IOException {
VirtualFile file = createTempFile("txt", null, THREE_RUSSIAN_LETTERS, CharsetToolkit.UTF8_CHARSET);
getDocument(file);
assertEquals("auto-detected from bytes", LoadTextUtil.wasCharsetDetectedFromBytes(file));
setContentOnDisk(new File(file.getPath()), null, THREE_RUSSIAN_LETTERS, WINDOWS_1251);
file.refresh(false, false);
Assert.assertNull(LoadTextUtil.wasCharsetDetectedFromBytes(file));
}
public void testSafeToConvert() throws IOException {
String text = "xxx";
VirtualFile file = createTempFile("txt", null, text, CharsetToolkit.UTF8_CHARSET);
byte[] bytes = file.contentsToByteArray();
assertEquals(EncodingUtil.Magic8.ABSOLUTELY, EncodingUtil.isSafeToConvertTo(file, text, bytes, US_ASCII));
assertEquals(EncodingUtil.Magic8.ABSOLUTELY, EncodingUtil.isSafeToConvertTo(file, text, bytes, WINDOWS_1251));
assertEquals(EncodingUtil.Magic8.WELL_IF_YOU_INSIST, EncodingUtil.isSafeToConvertTo(file, text, bytes, CharsetToolkit.UTF_16BE_CHARSET));
String rustext = THREE_RUSSIAN_LETTERS;
VirtualFile rusfile = createTempFile("txt", null, rustext, CharsetToolkit.UTF8_CHARSET);
byte[] rusbytes = rusfile.contentsToByteArray();
assertEquals(EncodingUtil.Magic8.NO_WAY, EncodingUtil.isSafeToConvertTo(rusfile, rustext, rusbytes, US_ASCII));
assertEquals(EncodingUtil.Magic8.WELL_IF_YOU_INSIST, EncodingUtil.isSafeToConvertTo(rusfile, rustext, rusbytes, WINDOWS_1251));
assertEquals(EncodingUtil.Magic8.WELL_IF_YOU_INSIST, EncodingUtil.isSafeToConvertTo(rusfile, rustext, rusbytes, CharsetToolkit.UTF_16BE_CHARSET));
String bomtext = THREE_RUSSIAN_LETTERS;
VirtualFile bomfile = createTempFile("txt", CharsetToolkit.UTF16LE_BOM, bomtext, CharsetToolkit.UTF_16LE_CHARSET);
byte[] bombytes = bomfile.contentsToByteArray();
assertEquals(EncodingUtil.Magic8.NO_WAY, EncodingUtil.isSafeToConvertTo(bomfile, bomtext, bombytes, US_ASCII));
assertEquals(EncodingUtil.Magic8.WELL_IF_YOU_INSIST, EncodingUtil.isSafeToConvertTo(bomfile, bomtext, bombytes, WINDOWS_1251));
assertEquals(EncodingUtil.Magic8.WELL_IF_YOU_INSIST, EncodingUtil.isSafeToConvertTo(bomfile, bomtext, bombytes, CharsetToolkit.UTF_16BE_CHARSET));
}
public void testSafeToReloadUtf8Bom() throws IOException {
String text = THREE_RUSSIAN_LETTERS;
VirtualFile file = createTempFile("txt", CharsetToolkit.UTF8_BOM, text, CharsetToolkit.UTF8_CHARSET);
byte[] bytes = file.contentsToByteArray();
assertEquals(EncodingUtil.Magic8.NO_WAY, EncodingUtil.isSafeToReloadIn(file, text, bytes, US_ASCII));
assertEquals(EncodingUtil.Magic8.NO_WAY, EncodingUtil.isSafeToReloadIn(file, text, bytes, CharsetToolkit.UTF_16LE_CHARSET));
}
public void testSafeToReloadUtf8NoBom() throws IOException {
String text = THREE_RUSSIAN_LETTERS;
VirtualFile file = createTempFile("txt", null, text, CharsetToolkit.UTF8_CHARSET);
byte[] bytes = file.contentsToByteArray();
assertEquals(EncodingUtil.Magic8.NO_WAY, EncodingUtil.isSafeToReloadIn(file, text, bytes, US_ASCII));
assertEquals(EncodingUtil.Magic8.NO_WAY, EncodingUtil.isSafeToReloadIn(file, text, bytes, CharsetToolkit.UTF_16LE_CHARSET));
}
public void testSafeToReloadText() throws IOException {
String text = "xxx";
VirtualFile file = createTempFile("txt", null, text, US_ASCII);
byte[] bytes = file.contentsToByteArray();
assertEquals(EncodingUtil.Magic8.ABSOLUTELY, EncodingUtil.isSafeToReloadIn(file, text, bytes, WINDOWS_1251));
assertEquals(EncodingUtil.Magic8.ABSOLUTELY, EncodingUtil.isSafeToReloadIn(file, text, bytes, CharsetToolkit.UTF8_CHARSET));
assertEquals(EncodingUtil.Magic8.NO_WAY, EncodingUtil.isSafeToReloadIn(file, text, bytes, CharsetToolkit.UTF_16LE_CHARSET));
}
public void testUndoChangeEncoding() throws IOException {
VirtualFile file = createTempFile("txt", null, "xxx", CharsetToolkit.UTF8_CHARSET);
file.setCharset(CharsetToolkit.UTF8_CHARSET);
UIUtil.dispatchAllInvocationEvents();
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
FileDocumentManager documentManager = FileDocumentManager.getInstance();
assertNull(documentManager.getCachedDocument(file));
change(file, WINDOWS_1251);
assertEquals(WINDOWS_1251, file.getCharset());
Document document = documentManager.getDocument(file);
change(file, CharsetToolkit.UTF8_CHARSET);
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
globalUndo();
UIUtil.dispatchAllInvocationEvents();
assertEquals(WINDOWS_1251, file.getCharset());
document.getText();
}
private void globalUndo() {
UndoManager myManager = UndoManager.getInstance(getProject());
assertTrue("undo is not available", myManager.isUndoAvailable(null));
myManager.undo(null);
}
private static void change(VirtualFile file, Charset charset) throws IOException {
new ChangeFileEncodingAction().chosen(getDocument(file), null, file, file.contentsToByteArray(), charset);
}
public void testCantReloadBOMDetected() throws IOException {
VirtualFile file = createTempFile("txt", CharsetToolkit.UTF8_BOM, THREE_RUSSIAN_LETTERS, CharsetToolkit.UTF8_CHARSET);
file.contentsToByteArray();
Document document = FileDocumentManager.getInstance().getDocument(file);
assertEquals(THREE_RUSSIAN_LETTERS, document.getText());
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
EncodingManager.getInstance().setEncoding(file, CharsetToolkit.UTF_16LE_CHARSET);
UIUtil.dispatchAllInvocationEvents();
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
}
public void testExternalChangeStrippedBOM() throws IOException {
String text = "text";
VirtualFile file = createTempFile("txt", CharsetToolkit.UTF8_BOM, text, CharsetToolkit.UTF8_CHARSET);
file.contentsToByteArray();
Document document = FileDocumentManager.getInstance().getDocument(file);
assertEquals(text, document.getText());
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
assertArrayEquals(CharsetToolkit.UTF8_BOM, file.getBOM());
FileUtil.writeToFile(VfsUtilCore.virtualToIoFile(file), text.getBytes(CharsetToolkit.UTF8_CHARSET));
file.refresh(false, false);
UIUtil.dispatchAllInvocationEvents();
assertEquals(text, document.getText());
assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset());
assertNull(file.getBOM());
}
public void testNewFileCreatedInProjectEncoding() throws Exception {
String oldIDE = EncodingManager.getInstance().getDefaultCharsetName();
EncodingManager.getInstance().setDefaultCharsetName("UTF-8");
String oldProject = EncodingProjectManager.getInstance(getProject()).getDefaultCharsetName();
EncodingProjectManager.getInstance(getProject()).setDefaultCharsetName(WINDOWS_1251.name());
try {
PsiFile psiFile = createFile("x.txt", "xx");
VirtualFile file = psiFile.getVirtualFile();
assertEquals(WINDOWS_1251, file.getCharset());
}
finally {
EncodingManager.getInstance().setDefaultCharsetName(oldIDE);
EncodingProjectManager.getInstance(getProject()).setDefaultCharsetName(oldProject);
}
}
public void testNewFileCreatedInProjectEncodingEvenIfItSetToDefault() throws Exception {
Charset defaultCharset = Charset.defaultCharset();
String oldIDE = EncodingManager.getInstance().getDefaultCharsetName();
EncodingManager.getInstance().setDefaultCharsetName(defaultCharset.name().equals("UTF-8") ? "windows-1251" : "UTF-8");
String oldProject = EncodingProjectManager.getInstance(getProject()).getDefaultCharsetName();
EncodingProjectManager.getInstance(getProject()).setDefaultCharsetName(""); // default charset
try {
PsiFile psiFile = createFile("x.txt", "xx");
VirtualFile file = psiFile.getVirtualFile();
assertEquals(EncodingManager.getInstance().getDefaultCharset(), file.getCharset());
}
finally {
EncodingManager.getInstance().setDefaultCharsetName(oldIDE);
EncodingProjectManager.getInstance(getProject()).setDefaultCharsetName(oldProject);
}
}
private PsiFile createFile(String fileName, String text) throws IOException {
File dir = createTempDirectory();
VirtualFile vDir = LocalFileSystem.getInstance().refreshAndFindFileByPath(dir.getCanonicalPath().replace(File.separatorChar, '/'));
assert vDir != null : dir;
return new WriteAction<PsiFile>() {
@Override
protected void run(@NotNull Result<PsiFile> result) throws Throwable {
if (!ModuleRootManager.getInstance(getModule()).getFileIndex().isInSourceContent(vDir)) {
PsiTestUtil.addSourceContentToRoots(getModule(), vDir);
}
final VirtualFile vFile = vDir.createChildData(vDir, fileName);
VfsUtil.saveText(vFile, text);
assertNotNull(vFile);
final PsiFile file = getPsiManager().findFile(vFile);
assertNotNull(file);
result.setResult(file);
}
}.execute().getResultObject();
}
public void testTheDefaultProjectEncodingIfNotSpecifiedShouldBeIDEEncoding() throws Exception {
String differentFromDefault = Charset.defaultCharset().name().equals("UTF-8") ? "windows-1251" : "UTF-8";
String oldIDE = EncodingManager.getInstance().getDefaultCharsetName();
try {
EncodingManager.getInstance().setDefaultCharsetName(differentFromDefault);
File temp = createTempDirectory();
VirtualFile tempDir = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(temp);
final Project newProject = ProjectManagerEx.getInstanceEx().newProject("new", tempDir.getPath(), false, false);
Disposer.register(getTestRootDisposable(), () -> ApplicationManager.getApplication().runWriteAction(() -> Disposer.dispose(newProject)));
PlatformTestUtil.saveProject(newProject);
Charset newProjectEncoding = EncodingProjectManager.getInstance(newProject).getDefaultCharset();
assertEquals(differentFromDefault, newProjectEncoding.name());
PsiFile psiFile = createFile("x.txt", "xx");
VirtualFile file = psiFile.getVirtualFile();
assertEquals(differentFromDefault, file.getCharset().name());
}
finally {
EncodingManager.getInstance().setDefaultCharsetName(oldIDE);
}
}
public void testFileMustNotLoadInWrongEncodingIfAccessedBeforeProjectOpen() {
VirtualFile dir = getTestRoot().findChild("newEncoding");
VirtualFile file = dir.findFileByRelativePath("src/xxx.txt");
Document document = FileDocumentManager.getInstance().getDocument(file);
document.getText();
UIUtil.dispatchAllInvocationEvents();
Project newEncodingProject = ProjectUtil.openProject(dir.getPath(), null, false);
UIUtil.dispatchAllInvocationEvents();
try {
assertEquals(US_ASCII, file.getCharset());
}
finally {
ProjectUtil.closeAndDispose(newEncodingProject);
}
}
public void testFileInsideJarCorrectlyHandlesBOM() throws IOException {
File tmpDir = createTempDirectory();
File jar = new File(tmpDir, "x.jar");
String text = "update";
byte[] bytes = ArrayUtil.mergeArrays(CharsetToolkit.UTF16BE_BOM, text.getBytes(CharsetToolkit.UTF_16BE_CHARSET));
String name = "sjkdhfksjdf";
IoTestUtil.createTestJar(jar, Collections.singletonList(Pair.create(name, bytes)));
VirtualFile vFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(jar);
assertNotNull(vFile);
VirtualFile jarRoot = JarFileSystem.getInstance().getJarRootForLocalFile(vFile);
assertNotNull(jarRoot);
VirtualFile file = jarRoot.findChild(name);
assertNotNull(file);
assertEquals(CharsetToolkit.UTF_16BE_CHARSET, file.getCharset());
assertEquals(PlainTextFileType.INSTANCE, file.getFileType());
assertEquals(text, LoadTextUtil.loadText(file).toString());
try (InputStream stream = file.getInputStream()) {
String loaded = new String(FileUtil.loadBytes(stream), CharsetToolkit.UTF_16BE_CHARSET);
assertEquals(text, loaded);
}
}
public void testBigFileInsideJarCorrectlyHandlesBOM() throws IOException {
File tmpDir = createTempDirectory();
File jar = new File(tmpDir, "x.jar");
String text = StringUtil.repeat("u", FileUtilRt.LARGE_FOR_CONTENT_LOADING+1);
byte[] bytes = ArrayUtil.mergeArrays(CharsetToolkit.UTF16BE_BOM, text.getBytes(CharsetToolkit.UTF_16BE_CHARSET));
String name = "sjkdhfksjdf";
IoTestUtil.createTestJar(jar, Collections.singletonList(Pair.create(name, bytes)));
VirtualFile vFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(jar);
assertNotNull(vFile);
VirtualFile jarRoot = JarFileSystem.getInstance().getJarRootForLocalFile(vFile);
assertNotNull(jarRoot);
VirtualFile file = jarRoot.findChild(name);
assertNotNull(file);
try (InputStream stream = file.getInputStream()) {
String loaded = new String(FileUtil.loadBytes(stream, 8192*2), CharsetToolkit.UTF_16BE_CHARSET);
assertEquals(text.substring(0, 8192), loaded);
}
assertEquals(PlainTextFileType.INSTANCE, file.getFileType());
assertEquals(CharsetToolkit.UTF_16BE_CHARSET, file.getCharset());
// should not load the entire file anyway
//assertEquals(text, LoadTextUtil.loadText(file).toString());
}
}