MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

handle incomplete ansi escape sequences correctly, input text can be cut in the middle of an escape sequence, because of standard streams buffer size (IDEA-169065)

Browse Source
This commit is contained in:
Sergey Simonchik
2017-03-13 18:03:40 +03:00
parent 849760597d
commit 2fb8b2db17
2 changed files with 235 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/platform-api/src/com/intellij/execution/process/AnsiEscapeDecoder.java
+105 -33
View File
@@ -36,8 +36,10 @@ public class AnsiEscapeDecoder {
private static final String M_CSI = "m" + CSI;
private static final char BACKSPACE = '\b';
private Key myCurrentTextAttributes;
private final ColoredOutputTypeRegistry myColoredOutputTypeRegistry = ColoredOutputTypeRegistry.getInstance();
private Key myCurrentTextAttributes;
private String myUnhandledStdout;
private String myUnhandledStderr;
/**
* Parses ansi-color codes from text and sends text fragments with color attributes to textAcceptor
@@ -48,31 +50,40 @@ public class AnsiEscapeDecoder {
* It can implement ColoredChunksAcceptor to receive list of pairs (text, attribute).
*/
public void escapeText(@NotNull String text, @NotNull Key outputType, @NotNull ColoredTextAcceptor textAcceptor) {
text = prependUnhandledText(text, outputType);
text = normalizeAsciiControlCharacters(text);
int pos = 0;
List<Pair<String, Key>> chunks = null;
int unhandledSuffixLength = 0;
while (true) {
int escSeqBeginInd = text.indexOf(CSI, pos);
int escSeqBeginInd = findEscSeqBeginIndex(text, pos);
if (escSeqBeginInd < 0) {
if (escSeqBeginInd < -1) {
unhandledSuffixLength = decodeUnhandledSuffixLength(escSeqBeginInd);
}
break;
}
if (pos < escSeqBeginInd) {
chunks = processTextChunk(chunks, text.substring(pos, escSeqBeginInd), outputType, textAcceptor);
}
final int escSeqEndInd = findEscSeqEndIndex(text, escSeqBeginInd);
int escSeqEndInd = findConsecutiveEscSequencesEndIndex(text, escSeqBeginInd);
if (escSeqEndInd < 0) {
if (escSeqEndInd < -1) {
unhandledSuffixLength = decodeUnhandledSuffixLength(escSeqEndInd);
}
break;
}
if (text.charAt(escSeqEndInd - 1) == 'm') {
String escSeq = text.substring(escSeqBeginInd, escSeqEndInd);
if (text.charAt(escSeqEndInd) == 'm') {
String escSeq = text.substring(escSeqBeginInd, escSeqEndInd + 1);
// this is a simple fix for RUBY-8996:
// we replace several consecutive escape sequences with one which contains all these sequences
String colorAttribute = StringUtil.replace(escSeq, M_CSI, ";");
myCurrentTextAttributes = myColoredOutputTypeRegistry.getOutputKey(colorAttribute);
}
pos = escSeqEndInd;
pos = escSeqEndInd + 1;
}
if (pos < text.length()) {
updateUnhandledSuffix(text, outputType, unhandledSuffixLength);
if (unhandledSuffixLength == 0 && pos < text.length()) {
chunks = processTextChunk(chunks, text.substring(pos), outputType, textAcceptor);
}
if (chunks != null && textAcceptor instanceof ColoredChunksAcceptor) {
@@ -80,6 +91,30 @@ public class AnsiEscapeDecoder {
}
}
private void updateUnhandledSuffix(@NotNull String text, @NotNull Key outputType, int unhandledSuffixLength) {
String unhandledSuffix = unhandledSuffixLength > 0 ? text.substring(text.length() - unhandledSuffixLength) : null;
if (outputType == ProcessOutputTypes.STDOUT) {
myUnhandledStdout = unhandledSuffix;
}
else if (outputType == ProcessOutputTypes.STDERR) {
myUnhandledStderr = unhandledSuffix;
}
}
@NotNull
private String prependUnhandledText(@NotNull String text, @NotNull Key outputType) {
String prevUnhandledText = null;
if (outputType == ProcessOutputTypes.STDOUT) {
prevUnhandledText = myUnhandledStdout;
myUnhandledStdout = null;
}
else if (outputType == ProcessOutputTypes.STDERR) {
prevUnhandledText = myUnhandledStderr;
myUnhandledStderr = null;
}
return prevUnhandledText != null ? prevUnhandledText + text : text;
}
@NotNull
private static String normalizeAsciiControlCharacters(@NotNull String text) {
int ind = text.indexOf(BACKSPACE);
@@ -122,29 +157,58 @@ public class AnsiEscapeDecoder {
return result.toString();
}
/*
* Selects all consecutive escape sequences and returns escape sequence end index (exclusive).
* If the escape sequence isn't finished, returns -1.
/**
* Returns the index of the first occurrence of CSI within the passed string that is greater than or equal to {@code fromIndex},
* or negative number if CSI is not found: -1 - (length of text suffix to keep in case of an incomplete CSI).
*/
private static int findEscSeqEndIndex(@NotNull String text, final int escSeqBeginInd) {
int beginInd = escSeqBeginInd;
while (true) {
int letterInd = findEscSeqLetterIndex(text, beginInd);
if (letterInd == -1) {
return beginInd == escSeqBeginInd ? -1 : beginInd;
}
if (text.charAt(letterInd) != 'm') {
return beginInd == escSeqBeginInd ? letterInd + 1 : beginInd;
}
beginInd = letterInd + 1;
}
}
private static int findEscSeqLetterIndex(@NotNull String text, int escSeqBeginInd) {
if (!text.regionMatches(escSeqBeginInd, CSI, 0, CSI.length())) {
private static int findEscSeqBeginIndex(@NotNull String text, int fromIndex) {
int ind = text.indexOf(CSI.charAt(0), fromIndex);
if (ind == -1) {
return -1;
}
int parameterEndInd = escSeqBeginInd + 2;
else if (ind == text.length() - 1) {
return encodeUnhandledSuffixLength(text, ind);
}
return text.charAt(ind + 1) == CSI.charAt(1) ? ind : -1;
}
/**
* Returns end index of all consecutive escape sequences started at {@code firstEscSeqBeginInd}, or
* negative number if not found: -1 - (length of string suffix to keep in case of an incomplete last escape sequence).
*/
private static int findConsecutiveEscSequencesEndIndex(@NotNull String text, int firstEscSeqBeginInd) {
int escSeqBeginInd = firstEscSeqBeginInd;
int lastMatchedColorEscSeqEndInd = -1;
int escSeqEndInd;
while ((escSeqEndInd = findEscSeqEndIndex(text, escSeqBeginInd)) >= 0) {
if (text.charAt(escSeqEndInd) != 'm') {
// Handle non-color escape sequences separately
// ColoredOutputTypeRegistry expects only color escape sequences and in a single consecutive text chunk
return lastMatchedColorEscSeqEndInd > 0 ? lastMatchedColorEscSeqEndInd : escSeqEndInd;
}
escSeqBeginInd = escSeqEndInd + 1;
lastMatchedColorEscSeqEndInd = escSeqEndInd;
if (escSeqEndInd + 1 >= text.length()) {
return encodeUnhandledSuffixLength(text, firstEscSeqBeginInd);
}
if (text.charAt(escSeqEndInd + 1) != CSI.charAt(0)) {
break;
}
if (escSeqEndInd + 2 >= text.length()) {
return encodeUnhandledSuffixLength(text, firstEscSeqBeginInd);
}
if (text.charAt(escSeqEndInd + 2) != CSI.charAt(1)) {
break;
}
}
if (escSeqEndInd < -1) {
return encodeUnhandledSuffixLength(text, firstEscSeqBeginInd);
}
return lastMatchedColorEscSeqEndInd;
}
private static int findEscSeqEndIndex(@NotNull String text, int escSeqBeginInd) {
int parameterEndInd = escSeqBeginInd + CSI.length();
while (parameterEndInd < text.length()) {
char ch = text.charAt(parameterEndInd);
if (Character.isDigit(ch) || ch == ';') {
@@ -154,13 +218,21 @@ public class AnsiEscapeDecoder {
break;
}
}
if (parameterEndInd < text.length()) {
char letter = text.charAt(parameterEndInd);
if (StringUtil.containsChar("ABCDEFGHJKSTfmisu", letter)) {
return parameterEndInd;
}
if (parameterEndInd == text.length()) {
return encodeUnhandledSuffixLength(text, escSeqBeginInd);
}
return -1;
return StringUtil.containsChar("ABCDEFGHJKSTfmisu", text.charAt(parameterEndInd)) ? parameterEndInd : -1;
}
private static int encodeUnhandledSuffixLength(@NotNull String text, int suffixStartInd) {
return -1 - (text.length() - suffixStartInd);
}
private static int decodeUnhandledSuffixLength(int encodedUnhandledSuffixLength) {
if (encodedUnhandledSuffixLength >= -1) {
throw new AssertionError();
}
return -encodedUnhandledSuffixLength - 1;
}
@Nullable
platform/platform-tests/testSrc/com/intellij/execution/process/AnsiEscapeDecoderTest.java
+130 -78
View File
@@ -1,8 +1,6 @@
package com.intellij.execution.process;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.BufferExposingByteArrayInputStream;
import com.intellij.testFramework.PlatformTestCase;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.util.Consumer;
@@ -10,101 +8,138 @@ import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collection;
import java.io.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class AnsiEscapeDecoderTest extends PlatformTestCase {
private static final String STDOUT_KEY = ProcessOutputTypes.STDOUT.toString();
private static final String STDERR_KEY = ProcessOutputTypes.STDERR.toString();
public void testTextWithoutColors() throws Exception {
AnsiEscapeDecoder decoder = new AnsiEscapeDecoder();
decoder.escapeText("", ProcessOutputTypes.STDOUT, createExpectedAcceptor(
Pair.create("", ProcessOutputTypes.STDOUT)
));
decoder.escapeText("simple text", ProcessOutputTypes.STDOUT, createExpectedAcceptor(
Pair.create("simple text", ProcessOutputTypes.STDOUT)
));
check(new ColoredText(""));
check(new ColoredText("simple text").addExpected("simple text", STDOUT_KEY));
}
public void testSingleColoredChunk() throws Exception {
AnsiEscapeDecoder decoder = new AnsiEscapeDecoder();
decoder.escapeText("Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n", ProcessOutputTypes.STDOUT, createExpectedAcceptor(
Pair.create("Chrome 35.0.1916 (Linux): Executed 0 of 1", ProcessOutputTypes.STDOUT),
Pair.create(" SUCCESS", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[32m")),
Pair.create(" (0 secs / 0 secs)\n", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[39m"))
));
check(new ColoredText("Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n")
.addExpected("Chrome 35.0.1916 (Linux): Executed 0 of 1", STDOUT_KEY)
.addExpected(" SUCCESS", "\u001B[32m")
.addExpected(" (0 secs / 0 secs)\n", "\u001B[39m"));
}
public void testCompoundEscSeq() throws Exception {
AnsiEscapeDecoder decoder = new AnsiEscapeDecoder();
decoder.escapeText("E\u001B[41m\u001B[37mE\u001B[0mE", ProcessOutputTypes.STDOUT, createExpectedAcceptor(
Pair.create("E", ProcessOutputTypes.STDOUT),
Pair.create("E", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[41;37m")),
Pair.create("E", ProcessOutputTypes.STDOUT)
));
check(new ColoredText("E\u001B[41m\u001B[37mE\u001B[0mE")
.addExpected("E", STDOUT_KEY)
.addExpected("E", "\u001B[41;37m")
.addExpected("E", STDOUT_KEY));
}
public void testOtherEscSeq() throws Exception {
AnsiEscapeDecoder decoder = new AnsiEscapeDecoder();
decoder.escapeText("Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m",
ProcessOutputTypes.STDOUT,
createExpectedAcceptor(
Pair.create("Plain", ProcessOutputTypes.STDOUT),
Pair.create("Green", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[32m")),
Pair.create("Normal", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[39m")),
Pair.create("Red", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[31m"))
)
);
check(new ColoredText("Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m")
.addExpected("Plain", STDOUT_KEY)
.addExpected("Green", "\u001B[32m")
.addExpected("Normal", "\u001B[39m")
.addExpected("Red", "\u001B[31m"));
}
public void testBackspaceControlSequence() throws Exception {
check(false, ContainerUtil.newArrayList(
new ColoredText(" 10% 0/1 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 70% 1/1 build modules")
.addExpected(" 70% 1/1 build modules", STDOUT_KEY),
new ColoredText(
"\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 40% 1/2 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 30% 1/3 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 25% 1/4 build modules")
.addExpected("\n 25% 1/4 build modules", STDOUT_KEY)
));
}
public void testIncompleteEscapeSequences() throws Exception {
check(true, ContainerUtil.newArrayList(
new ColoredText("\u001B"),
new ColoredText("[33m Hello\u001B[3").addExpected(" Hello", "\u001B[33m"),
new ColoredText("4m, Work!").addExpected(", Work!", "\u001B[34m")
));
check(true, ContainerUtil.newArrayList(
new ColoredText("\u001B[1;33m<\u001B[34mnamespace\u001B[1")
.addExpected("<", "\u001B[1;33m")
.addExpected("namespace", "\u001B[34m"),
new ColoredText(
";33m:abcd\u001B[0m\u001B[1;33m>\u001B[0m0\u001B[1;33m</\u001B[34mnamespace\u001B[1;33m:abcd\u001B[0m\u001B[1;33m>\u001B[0m")
.addExpected(":abcd", "\u001B[1;33m")
.addExpected(">", "\u001B[0;1;33m")
.addExpected("0", "stdout")
.addExpected("</", "\u001B[1;33m")
.addExpected("namespace", "\u001B[34m")
.addExpected(":abcd", "\u001B[1;33m")
.addExpected(">", "\u001B[0;1;33m")
));
}
private static void check(@NotNull ColoredText text) {
check(true, Collections.singletonList(text));
}
private static void check(boolean testCharByCharProcessing, @NotNull List<ColoredText> texts) {
AnsiEscapeDecoder decoder = new AnsiEscapeDecoder();
decoder.escapeText(" 10% 0/1 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 70% 1/1 build modules",
ProcessOutputTypes.STDERR,
createExpectedAcceptor(
Pair.create(" 70% 1/1 build modules", ProcessOutputTypes.STDERR)
)
);
decoder.escapeText("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 40% 1/2 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 30% 1/3 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 25% 1/4 build modules",
ProcessOutputTypes.STDERR,
createExpectedAcceptor(
Pair.create("\n 25% 1/4 build modules", ProcessOutputTypes.STDERR)
)
);
}
@NotNull
private static List<Pair<String, String>> toListWithKeyName(@NotNull Collection<Pair<String, Key>> list) {
return ContainerUtil.map(list, pair -> Pair.create(pair.first, pair.second.toString()));
}
@SafeVarargs
private static AnsiEscapeDecoder.ColoredChunksAcceptor createExpectedAcceptor(@NotNull final Pair<String, Key>... expected) {
return new AnsiEscapeDecoder.ColoredChunksAcceptor() {
@Override
public void coloredChunksAvailable(@NotNull List<Pair<String, Key>> chunks) {
List<Pair<String, String>> expectedWithKeyName = toListWithKeyName(Arrays.asList(expected));
List<Pair<String, String>> actualWithKeyName = toListWithKeyName(chunks);
Assert.assertEquals(expectedWithKeyName, actualWithKeyName);
}
@Override
public void coloredTextAvailable(@NotNull String text, @NotNull Key attributes) {
throw new RuntimeException(); // shouldn't be called
}
List<Pair<String, String>> actualColoredChunks = ContainerUtil.newArrayList();
//noinspection CodeBlock2Expr
AnsiEscapeDecoder.ColoredTextAcceptor acceptor = (text, attributes) -> {
actualColoredChunks.add(Pair.create(text, attributes.toString()));
};
// test stdout
for (ColoredText text : texts) {
decoder.escapeText(text.myRawText, ProcessOutputTypes.STDOUT, acceptor);
}
List<Pair<String, String>> expectedColoredChunks = new ArrayList<>();
for (ColoredText text : texts) {
expectedColoredChunks.addAll(text.myExpectedColoredChunks);
}
Assert.assertEquals(expectedColoredChunks, actualColoredChunks);
if (testCharByCharProcessing) {
// test stdout char by char
actualColoredChunks.clear();
decoder = new AnsiEscapeDecoder();
for (ColoredText text : texts) {
for (int i = 0; i < text.myRawText.length(); i++) {
decoder.escapeText(String.valueOf(text.myRawText.charAt(i)), ProcessOutputTypes.STDOUT, acceptor);
}
}
expectedColoredChunks.clear();
for (ColoredText text : texts) {
for (Pair<String, String> chunk : text.myExpectedColoredChunks) {
String chunkText = chunk.first;
for (int i = 0; i < chunkText.length(); i++) {
expectedColoredChunks.add(Pair.create(String.valueOf(chunkText.charAt(i)), chunk.second));
}
}
}
Assert.assertEquals(expectedColoredChunks, actualColoredChunks);
}
// test stderr
actualColoredChunks.clear();
decoder = new AnsiEscapeDecoder();
for (ColoredText text : texts) {
decoder.escapeText(text.myRawText, ProcessOutputTypes.STDERR, acceptor);
}
expectedColoredChunks.clear();
for (ColoredText text : texts) {
for (Pair<String, String> chunk : text.myExpectedColoredChunks) {
expectedColoredChunks.add(Pair.create(chunk.first, STDERR_KEY));
}
}
Assert.assertEquals(expectedColoredChunks, actualColoredChunks);
}
@NotNull
public static Process createTestProcess() {
// have to be synchronised because used from pooled thread
ByteArrayOutputStream outputStream = new ByteArrayOutputStream(10000);
BufferExposingByteArrayInputStream inputStream = new BufferExposingByteArrayInputStream(new byte[0]);
AtomicBoolean finished = new AtomicBoolean();
return new Process() {
@Override
@@ -114,17 +149,17 @@ public class AnsiEscapeDecoderTest extends PlatformTestCase {
@Override
public InputStream getInputStream() {
return inputStream;
return new ByteArrayInputStream(new byte[0]);
}
@Override
public InputStream getErrorStream() {
return inputStream;
return new ByteArrayInputStream(new byte[0]);
}
@Override
public int waitFor() {
while (!finished.get());
while (!finished.get()) {}
return 0;
}
@@ -143,11 +178,14 @@ public class AnsiEscapeDecoderTest extends PlatformTestCase {
public void testPerformance() throws IOException {
Process testProcess = createTestProcess();
//noinspection CodeBlock2Expr
withProcessHandlerFrom(testProcess, handler -> {
PlatformTestUtil.startPerformanceTest("ansi color", 15000, ()->{
for (int i=0; i<2_000_000;i++) {
handler.notifyTextAvailable(i+"Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n", ProcessOutputTypes.STDOUT);
handler.notifyTextAvailable(i+"Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m\n", ProcessOutputTypes.SYSTEM);
PlatformTestUtil.startPerformanceTest("ansi color", 15000, () -> {
for (int i = 0; i < 2_000_000; i++) {
handler.notifyTextAvailable(i + "Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n",
ProcessOutputTypes.STDOUT);
handler.notifyTextAvailable(i + "Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m\n",
ProcessOutputTypes.SYSTEM);
}
}).cpuBound().assertTiming();
});
@@ -168,4 +206,18 @@ public class AnsiEscapeDecoderTest extends PlatformTestCase {
handler.waitFor();
}
}
private static class ColoredText {
private final String myRawText;
private final List<Pair<String, String>> myExpectedColoredChunks = new ArrayList<>();
public ColoredText(@NotNull String rawText) {
myRawText = rawText;
}
private ColoredText addExpected(@NotNull String text, @NotNull String colorKey) {
myExpectedColoredChunks.add(Pair.create(text, colorKey));
return this;
}
}
}