use Charsets.UTF_8

This commit is contained in:
Vladimir Krivosheev
2019-04-12 09:01:56 +02:00
parent c4355ed008
commit 40be3c10c1
18 changed files with 36 additions and 82 deletions
@@ -1,8 +1,7 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.references
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.vfs.CharsetToolkit
import com.intellij.util.PathUtil
import com.intellij.util.indexing.IndexId
import com.intellij.util.indexing.impl.MapIndexStorage
@@ -45,11 +44,11 @@ abstract class ReferenceIndexTestBase : JpsBuildTestCase() {
}
protected fun changeFileContent(name: String, changesSourceFile: String) {
changeFile("m/" + name, FileUtil.loadFile(File(testDataRootPath + "/" + getTestName(true) + "/" + changesSourceFile), CharsetToolkit.UTF8_CHARSET))
changeFile("m/" + name, FileUtil.loadFile(File(testDataRootPath + "/" + getTestName(true) + "/" + changesSourceFile), Charsets.UTF_8))
}
protected fun addFile(name: String): String {
return createFile("m/" + name, FileUtil.loadFile(File(getTestDataPath() + name), CharsetToolkit.UTF8_CHARSET))
return createFile("m/" + name, FileUtil.loadFile(File(getTestDataPath() + name), Charsets.UTF_8))
}
@@ -12,7 +12,6 @@ import org.jetbrains.jps.model.java.JavaSourceRootType
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
import java.io.File
import java.nio.charset.StandardCharsets
import java.nio.file.*
import java.util.*
import java.util.concurrent.atomic.AtomicInteger
@@ -104,7 +103,7 @@ class IconsClassGenerator(private val projectHome: File, val util: JpsModule, pr
val iconsClassInfo = getIconsClassInfo(module) ?: return
val outFile = iconsClassInfo.outFile
val oldText = try {
Files.readAllBytes(outFile).toString(StandardCharsets.UTF_8)
Files.readAllBytes(outFile).toString(Charsets.UTF_8)
}
catch (ignored: NoSuchFileException) {
null
@@ -24,7 +24,6 @@ import com.intellij.util.io.systemIndependentPath
import org.jdom.Element
import org.jdom.JDOMException
import java.io.IOException
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.NoSuchFileException
import java.nio.file.Path
@@ -129,7 +128,7 @@ open class FileBasedStorage(file: Path,
if (result == null) {
result = LocalFileSystem.getInstance().findFileByPath(file.systemIndependentPath)
// otherwise virtualFile.contentsToByteArray() will query expensive FileTypeManager.getInstance()).getByFile()
result?.charset = StandardCharsets.UTF_8
result?.charset = Charsets.UTF_8
cachedVirtualFile = result
}
return cachedVirtualFile
@@ -235,7 +234,7 @@ internal fun writeFile(file: Path?,
if ((LOG.isDebugEnabled || ApplicationManager.getApplication().isUnitTestMode) && !FileUtilRt.isTooLarge(result.length)) {
val content = dataWriter.toBufferExposingByteArray(lineSeparator)
if (isEqualContent(result, lineSeparator, content, prependXmlProlog)) {
val contentString = content.toByteArray().toString(StandardCharsets.UTF_8)
val contentString = content.toByteArray().toString(Charsets.UTF_8)
LOG.warn("Content equals, but it must be handled not on this level: file ${result.name}, content:\n$contentString")
}
else if (DEBUG_LOG != null && ApplicationManager.getApplication().isUnitTestMode) {
@@ -15,7 +15,6 @@ import com.intellij.util.LineSeparator
import org.jdom.Element
import java.io.FileNotFoundException
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import java.nio.file.Paths
@@ -98,7 +97,7 @@ internal class ModuleStateStorageManager(macroSubstitutor: TrackingPathMacroSubs
}
else {
runAndHandleExceptions {
val charBuffer = StandardCharsets.UTF_8.decode(ByteBuffer.wrap(virtualFile.contentsToByteArray()))
val charBuffer = Charsets.UTF_8.decode(ByteBuffer.wrap(virtualFile.contentsToByteArray()))
lineSeparator = detectLineSeparators(charBuffer, if (isUseXmlProlog) null else LineSeparator.LF)
return JDOMUtil.load(charBuffer)
}
@@ -15,7 +15,6 @@ import com.intellij.openapi.util.registry.Registry
import com.intellij.openapi.util.text.StringUtil
import java.io.*
import java.nio.charset.Charset
import java.nio.charset.StandardCharsets
object ExecUtil {
private val hasGkSudo = PathExecLazyValue("gksudo")
@@ -47,7 +46,7 @@ object ExecUtil {
fun loadTemplate(loader: ClassLoader, templateName: String, variables: Map<String, String>?): String {
val stream = loader.getResourceAsStream(templateName) ?: throw IOException("Template '$templateName' not found by $loader")
val template = FileUtil.loadTextAndClose(InputStreamReader(stream, StandardCharsets.UTF_8))
val template = FileUtil.loadTextAndClose(InputStreamReader(stream, Charsets.UTF_8))
if (variables == null || variables.isEmpty()) {
return template
}
@@ -67,7 +66,7 @@ object ExecUtil {
fun createTempExecutableScript(prefix: String, suffix: String, content: String): File {
val tempDir = File(PathManager.getTempPath())
val tempFile = FileUtil.createTempFile(tempDir, prefix, suffix, true, true)
FileUtil.writeToFile(tempFile, content.toByteArray(StandardCharsets.UTF_8))
FileUtil.writeToFile(tempFile, content.toByteArray(Charsets.UTF_8))
if (!tempFile.setExecutable(true, true)) {
throw ExecutionException("Failed to make temp file executable: $tempFile")
}
@@ -1,4 +1,4 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util.io
import com.intellij.openapi.progress.ProcessCanceledException
@@ -10,7 +10,6 @@ import java.io.IOException
import java.io.InputStream
import java.net.URLConnection
import java.nio.charset.Charset
import java.nio.charset.StandardCharsets
import java.util.regex.Pattern
private const val BLOCK_SIZE = 16 * 1024
@@ -49,6 +48,6 @@ internal object HttpUrlConnectionUtil {
}
}
}
return StandardCharsets.UTF_8
return Charsets.UTF_8
}
}
@@ -1,4 +1,4 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.internal.statistic.eventLog
import com.google.common.hash.Hashing
@@ -9,7 +9,6 @@ import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.util.BuildNumber
import com.intellij.openapi.util.text.StringUtil
import com.intellij.util.containers.ContainerUtil
import java.nio.charset.StandardCharsets
import java.security.SecureRandom
import java.util.*
import java.util.prefs.Preferences
@@ -40,7 +39,7 @@ object EventLogConfiguration {
val hasher = Hashing.sha256().newHasher()
hasher.putBytes(salt)
hasher.putString(data, StandardCharsets.UTF_8)
hasher.putString(data, Charsets.UTF_8)
val result = hasher.hash().toString()
anonymizedCache[data] = result
return result
@@ -1,4 +1,4 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.io
import io.netty.buffer.ByteBuf
@@ -8,10 +8,9 @@ import java.nio.ByteBuffer
import java.nio.CharBuffer
import java.nio.charset.CharacterCodingException
import java.nio.charset.CharsetDecoder
import java.nio.charset.StandardCharsets
fun ByteBuf.readIntoCharBuffer(byteCount: Int = readableBytes(), charBuffer: CharBuffer) {
val decoder = CharsetUtil.decoder(StandardCharsets.UTF_8)
val decoder = CharsetUtil.decoder(Charsets.UTF_8)
if (nioBufferCount() == 1) {
decodeString(decoder, internalNioBuffer(readerIndex(), byteCount), charBuffer)
}
@@ -1,22 +1,7 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.jsonProtocol
import com.google.gson.stream.JsonWriter
import com.intellij.openapi.vfs.CharsetToolkit
import com.intellij.util.containers.isNullOrEmpty
import com.intellij.util.io.writeUtf8
import gnu.trove.TIntArrayList
@@ -138,7 +123,7 @@ open class OutMessage() {
}
catch (e: IllegalStateException) {
if ("Nesting problem." == e.message) {
throw RuntimeException(item.buffer.toString(CharsetToolkit.UTF8_CHARSET) + "\nparent:\n" + buffer.toString(CharsetToolkit.UTF8_CHARSET), e)
throw RuntimeException(item.buffer.toString(Charsets.UTF_8) + "\nparent:\n" + buffer.toString(Charsets.UTF_8), e)
}
else {
throw e
@@ -16,7 +16,6 @@ import com.jetbrains.jsonSchema.extension.JsonSchemaFileProvider
import com.jetbrains.jsonSchema.extension.JsonSchemaProviderFactory
import com.jetbrains.jsonSchema.extension.SchemaType
import com.jetbrains.jsonSchema.impl.JsonSchemaVersion
import java.nio.charset.StandardCharsets
internal val LOG = logger<IntellijConfigurationJsonSchemaProviderFactory>()
@@ -42,7 +41,7 @@ internal class IntellijConfigurationJsonSchemaProviderFactory : JsonSchemaProvid
inner class MyJsonSchemaFileProvider : JsonSchemaFileProvider, DumbAware {
private val schemeFile = lazy {
LightVirtualFile("scheme.json", JsonFileType.INSTANCE, schemeContent, StandardCharsets.UTF_8, 0)
LightVirtualFile("scheme.json", JsonFileType.INSTANCE, schemeContent, Charsets.UTF_8, 0)
}
override fun getName() = "IntelliJ Configuration"
@@ -10,14 +10,13 @@ import com.intellij.testFramework.assertions.Assertions.assertThat
import com.jetbrains.jsonSchema.impl.JsonSchemaCompletionContributor
import com.jetbrains.jsonSchema.impl.JsonSchemaReader
import org.intellij.lang.annotations.Language
import java.nio.charset.StandardCharsets
// this test requires YamlJsonSchemaCompletionContributor, that's why intellij.yaml is added as test dependency
internal class ConfigurationSchemaTest : CompletionTestCase() {
private var schemaFile: LightVirtualFile? = null
override fun setUp() {
super.setUp()
schemaFile = LightVirtualFile("scheme.json", JsonFileType.INSTANCE, generateConfigurationSchema(), StandardCharsets.UTF_8, 0)
schemaFile = LightVirtualFile("scheme.json", JsonFileType.INSTANCE, generateConfigurationSchema(), Charsets.UTF_8, 0)
}
fun `test map and description`() {
@@ -20,7 +20,6 @@ import com.intellij.ui.EditorNotificationPanel
import com.intellij.ui.EditorNotifications
import com.intellij.util.ArrayUtil
import org.editorconfig.language.filetype.EditorConfigFileConstants
import java.nio.charset.StandardCharsets
import java.util.*
class EditorConfigWrongFileEncodingNotificationProvider : EditorNotifications.Provider<EditorNotificationPanel>(), DumbAware {
@@ -32,7 +31,7 @@ class EditorConfigWrongFileEncodingNotificationProvider : EditorNotifications.Pr
if (editor.getUserData(HIDDEN_KEY) != null) return null
if (PropertiesComponent.getInstance().isTrueValue(DISABLE_KEY)) return null
if (file.extension != EditorConfigFileConstants.FILE_EXTENSION) return null
if (file.charset == StandardCharsets.UTF_8) return null
if (file.charset == Charsets.UTF_8) return null
return buildPanel(project, editor, file)
}
@@ -45,7 +44,7 @@ class EditorConfigWrongFileEncodingNotificationProvider : EditorNotifications.Pr
val text = editor.document.text
val isSafeToConvert = isSafeToConvertTo(file, text)
val isSafeToReload = isSafeToReloadIn(file, text)
ChangeFileEncodingAction.changeTo(project, editor.document, editor, file, StandardCharsets.UTF_8, isSafeToConvert, isSafeToReload)
ChangeFileEncodingAction.changeTo(project, editor.document, editor, file, Charsets.UTF_8, isSafeToConvert, isSafeToReload)
}
val hide = EditorConfigBundle["notification.action.hide.once"]
@@ -68,17 +67,17 @@ class EditorConfigWrongFileEncodingNotificationProvider : EditorNotifications.Pr
val bytes = file.contentsToByteArray()
// file has BOM but the charset hasn't
val bom = file.bom
if (bom != null && !CharsetToolkit.canHaveBom(StandardCharsets.UTF_8, bom)) return EncodingUtil.Magic8.NO_WAY
if (bom != null && !CharsetToolkit.canHaveBom(Charsets.UTF_8, bom)) return EncodingUtil.Magic8.NO_WAY
// the charset has mandatory BOM (e.g. UTF-xx) but the file hasn't or has wrong
val mandatoryBom = CharsetToolkit.getMandatoryBom(StandardCharsets.UTF_8)
val mandatoryBom = CharsetToolkit.getMandatoryBom(Charsets.UTF_8)
if (mandatoryBom != null && !ArrayUtil.startsWith(bytes, mandatoryBom)) return EncodingUtil.Magic8.NO_WAY
val loaded = LoadTextUtil.getTextByBinaryPresentation(bytes, StandardCharsets.UTF_8).toString()
val loaded = LoadTextUtil.getTextByBinaryPresentation(bytes, Charsets.UTF_8).toString()
var bytesToSave: ByteArray = try {
val separator = FileDocumentManager.getInstance().getLineSeparator(file, null)
StringUtil.convertLineSeparators(loaded, separator).toByteArray(StandardCharsets.UTF_8)
StringUtil.convertLineSeparators(loaded, separator).toByteArray(Charsets.UTF_8)
}
catch (e: UnsupportedOperationException) {
return EncodingUtil.Magic8.NO_WAY
@@ -100,8 +99,8 @@ class EditorConfigWrongFileEncodingNotificationProvider : EditorNotifications.Pr
val bytesOnDisk = file.contentsToByteArray()
val lineSeparator = FileDocumentManager.getInstance().getLineSeparator(file, null)
val textToSave = if (lineSeparator == "\n") text else StringUtilRt.convertLineSeparators(text, lineSeparator)
val saved = LoadTextUtil.chooseMostlyHarmlessCharset(file.charset, StandardCharsets.UTF_8, textToSave.toString()).second
val textLoadedBack = LoadTextUtil.getTextByBinaryPresentation(saved, StandardCharsets.UTF_8)
val saved = LoadTextUtil.chooseMostlyHarmlessCharset(file.charset, Charsets.UTF_8, textToSave.toString()).second
val textLoadedBack = LoadTextUtil.getTextByBinaryPresentation(saved, Charsets.UTF_8)
when {
!StringUtil.equals(text, textLoadedBack) -> EncodingUtil.Magic8.NO_WAY
Arrays.equals(saved, bytesOnDisk) -> EncodingUtil.Magic8.ABSOLUTELY
@@ -1,25 +1,10 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package git4idea.index
import com.intellij.openapi.util.SystemInfo
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.vcs.Executor.*
import com.intellij.openapi.vcs.FilePath
import com.intellij.openapi.vfs.CharsetToolkit
import com.intellij.vcsUtil.VcsUtil
import git4idea.repo.GitRepository
import git4idea.test.GitPlatformTest
@@ -109,11 +94,11 @@ class GitIndexTest : GitPlatformTest() {
private fun readFileContent(path: String): String {
val stagedFile = GitIndexUtil.listStaged(repository, path.path)
val bytes = GitIndexUtil.read(repository, stagedFile!!.blobHash)
return String(bytes, CharsetToolkit.UTF8_CHARSET)
return String(bytes, Charsets.UTF_8)
}
private fun writeFileContent(path: String, content: String, executable: Boolean = false) {
val bytes = content.toByteArray(CharsetToolkit.UTF8_CHARSET)
val bytes = content.toByteArray(Charsets.UTF_8)
GitIndexUtil.write(repository, path.path, bytes, executable)
}
@@ -12,7 +12,6 @@ import com.intellij.openapi.vcs.changes.ChangesUtil
import com.intellij.openapi.vcs.changes.LocalChangeList
import com.intellij.openapi.vcs.ex.PartialLocalLineStatusTracker
import com.intellij.openapi.vcs.impl.LineStatusTrackerManager
import com.intellij.openapi.vfs.CharsetToolkit
import com.intellij.openapi.vfs.LocalFileSystem
import com.intellij.util.ui.UIUtil
import git4idea.test.*
@@ -203,7 +202,7 @@ class GitPartialCommitTest : GitSingleRepoTest() {
val actualContent = repo.gitAsBytes("cat-file" +
(if (useFilters) " --filters" else " -p") +
" :$fileName")
assertEquals(expectedContent, String(actualContent, CharsetToolkit.UTF8_CHARSET))
assertEquals(expectedContent, String(actualContent, Charsets.UTF_8))
}
private fun withTrackedDocument(fileName: String, newContent: String, task: (Document, PartialLocalLineStatusTracker) -> Unit) {
@@ -10,7 +10,6 @@ import com.intellij.openapi.fileTypes.StdFileTypes
import com.intellij.openapi.vcs.merge.MergeDialogCustomizer
import com.intellij.openapi.vcs.merge.MergeProvider2
import com.intellij.openapi.vcs.merge.MultipleFileMergeDialog
import com.intellij.openapi.vfs.CharsetToolkit
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.testFramework.LightVirtualFile
import com.intellij.util.PathUtilRt
@@ -164,7 +163,7 @@ suspend fun resolveConflicts(files: List<VirtualFile>, mergeProvider: MergeProvi
return processedFiles!!
}
class RepositoryVirtualFile(private val path: String) : LightVirtualFile(PathUtilRt.getFileName(path), StdFileTypes.XML, "", CharsetToolkit.UTF8_CHARSET, 1L) {
class RepositoryVirtualFile(private val path: String) : LightVirtualFile(PathUtilRt.getFileName(path), StdFileTypes.XML, "", Charsets.UTF_8, 1L) {
var byteContent: ByteArray? = null
private set
@@ -15,7 +15,6 @@ import org.eclipse.jgit.lib.FileMode
import org.eclipse.jgit.lib.Repository
import java.io.File
import java.io.FileInputStream
import java.nio.charset.StandardCharsets
import java.text.MessageFormat
import java.util.*
@@ -121,7 +120,7 @@ interface PathEdit {
abstract class PathEditBase(final override val path: ByteArray) : PathEdit
private fun encodePath(path: String): ByteArray {
val bytes = StandardCharsets.UTF_8.encode(path).toByteArray()
val bytes = Charsets.UTF_8.encode(path).toByteArray()
if (SystemInfo.isWindows) {
for (i in 0 until bytes.size) {
if (bytes[i].toChar() == '\\') {
@@ -20,7 +20,6 @@ import org.jetbrains.settingsRepository.git.computeIndexDiff
import org.jetbrains.settingsRepository.git.deletePath
import org.jetbrains.settingsRepository.git.writePath
import org.junit.Test
import java.nio.charset.StandardCharsets
import java.util.*
val MARKER_ACCEPT_MY = "__accept my__".toByteArray()
@@ -189,7 +188,7 @@ internal class GitTest : GitTestCase() {
sync(SyncType.MERGE)
restoreRemoteAfterPush()
fs.file(SAMPLE_FILE_NAME, data.toString(StandardCharsets.UTF_8)).compare()
fs.file(SAMPLE_FILE_NAME, data.toString(Charsets.UTF_8)).compare()
}
@Test fun `merge - theirs file deleted, my modified, accept theirs`() = runBlocking<Unit> {
@@ -1,10 +1,9 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.rest
import com.google.common.collect.Lists
import com.intellij.openapi.project.Project
import com.intellij.openapi.roots.ProjectFileIndex
import com.intellij.openapi.vfs.CharsetToolkit
import com.intellij.openapi.vfs.VirtualFile
import com.jetbrains.python.PythonHelper.REST_RUNNER
import com.jetbrains.python.sdk.PySdkUtil
@@ -19,7 +18,7 @@ class RestPythonPreviewProvider : RestPreviewProvider() {
val sdk = PythonSdkType.findPythonSdk(module) ?: return Pair("", "No Python interpreter configured for the project.")
val commandLine = REST_RUNNER.newCommandLine(sdk, Lists.newArrayList("rst2html"))
val output = PySdkUtil.getProcessOutput(commandLine, virtualFile.parent.path, null, 5000,
text.toByteArray(CharsetToolkit.UTF8_CHARSET), false)
text.toByteArray(Charsets.UTF_8), false)
return if (output.isCancelled || output.isTimeout)
Pair("", "Failed to generate html.")
else Pair(output.stdout, "<h3>Error output:</h3>" + output.stderr)