diff --git a/platform/platform-api/src/com/intellij/execution/process/AnsiEscapeDecoder.java b/platform/platform-api/src/com/intellij/execution/process/AnsiEscapeDecoder.java index 3f0030cef70e..a21170eb39ac 100644 --- a/platform/platform-api/src/com/intellij/execution/process/AnsiEscapeDecoder.java +++ b/platform/platform-api/src/com/intellij/execution/process/AnsiEscapeDecoder.java @@ -36,8 +36,10 @@ public class AnsiEscapeDecoder { private static final String M_CSI = "m" + CSI; private static final char BACKSPACE = '\b'; - private Key myCurrentTextAttributes; private final ColoredOutputTypeRegistry myColoredOutputTypeRegistry = ColoredOutputTypeRegistry.getInstance(); + private Key myCurrentTextAttributes; + private String myUnhandledStdout; + private String myUnhandledStderr; /** * Parses ansi-color codes from text and sends text fragments with color attributes to textAcceptor @@ -48,31 +50,40 @@ public class AnsiEscapeDecoder { * It can implement ColoredChunksAcceptor to receive list of pairs (text, attribute). */ public void escapeText(@NotNull String text, @NotNull Key outputType, @NotNull ColoredTextAcceptor textAcceptor) { + text = prependUnhandledText(text, outputType); text = normalizeAsciiControlCharacters(text); int pos = 0; List> chunks = null; + int unhandledSuffixLength = 0; while (true) { - int escSeqBeginInd = text.indexOf(CSI, pos); + int escSeqBeginInd = findEscSeqBeginIndex(text, pos); if (escSeqBeginInd < 0) { + if (escSeqBeginInd < -1) { + unhandledSuffixLength = decodeUnhandledSuffixLength(escSeqBeginInd); + } break; } if (pos < escSeqBeginInd) { chunks = processTextChunk(chunks, text.substring(pos, escSeqBeginInd), outputType, textAcceptor); } - final int escSeqEndInd = findEscSeqEndIndex(text, escSeqBeginInd); + int escSeqEndInd = findConsecutiveEscSequencesEndIndex(text, escSeqBeginInd); if (escSeqEndInd < 0) { + if (escSeqEndInd < -1) { + unhandledSuffixLength = decodeUnhandledSuffixLength(escSeqEndInd); + } break; } - if (text.charAt(escSeqEndInd - 1) == 'm') { - String escSeq = text.substring(escSeqBeginInd, escSeqEndInd); + if (text.charAt(escSeqEndInd) == 'm') { + String escSeq = text.substring(escSeqBeginInd, escSeqEndInd + 1); // this is a simple fix for RUBY-8996: // we replace several consecutive escape sequences with one which contains all these sequences String colorAttribute = StringUtil.replace(escSeq, M_CSI, ";"); myCurrentTextAttributes = myColoredOutputTypeRegistry.getOutputKey(colorAttribute); } - pos = escSeqEndInd; + pos = escSeqEndInd + 1; } - if (pos < text.length()) { + updateUnhandledSuffix(text, outputType, unhandledSuffixLength); + if (unhandledSuffixLength == 0 && pos < text.length()) { chunks = processTextChunk(chunks, text.substring(pos), outputType, textAcceptor); } if (chunks != null && textAcceptor instanceof ColoredChunksAcceptor) { @@ -80,6 +91,30 @@ public class AnsiEscapeDecoder { } } + private void updateUnhandledSuffix(@NotNull String text, @NotNull Key outputType, int unhandledSuffixLength) { + String unhandledSuffix = unhandledSuffixLength > 0 ? text.substring(text.length() - unhandledSuffixLength) : null; + if (outputType == ProcessOutputTypes.STDOUT) { + myUnhandledStdout = unhandledSuffix; + } + else if (outputType == ProcessOutputTypes.STDERR) { + myUnhandledStderr = unhandledSuffix; + } + } + + @NotNull + private String prependUnhandledText(@NotNull String text, @NotNull Key outputType) { + String prevUnhandledText = null; + if (outputType == ProcessOutputTypes.STDOUT) { + prevUnhandledText = myUnhandledStdout; + myUnhandledStdout = null; + } + else if (outputType == ProcessOutputTypes.STDERR) { + prevUnhandledText = myUnhandledStderr; + myUnhandledStderr = null; + } + return prevUnhandledText != null ? prevUnhandledText + text : text; + } + @NotNull private static String normalizeAsciiControlCharacters(@NotNull String text) { int ind = text.indexOf(BACKSPACE); @@ -122,29 +157,58 @@ public class AnsiEscapeDecoder { return result.toString(); } - /* - * Selects all consecutive escape sequences and returns escape sequence end index (exclusive). - * If the escape sequence isn't finished, returns -1. + /** + * Returns the index of the first occurrence of CSI within the passed string that is greater than or equal to {@code fromIndex}, + * or negative number if CSI is not found: -1 - (length of text suffix to keep in case of an incomplete CSI). */ - private static int findEscSeqEndIndex(@NotNull String text, final int escSeqBeginInd) { - int beginInd = escSeqBeginInd; - while (true) { - int letterInd = findEscSeqLetterIndex(text, beginInd); - if (letterInd == -1) { - return beginInd == escSeqBeginInd ? -1 : beginInd; - } - if (text.charAt(letterInd) != 'm') { - return beginInd == escSeqBeginInd ? letterInd + 1 : beginInd; - } - beginInd = letterInd + 1; - } - } - - private static int findEscSeqLetterIndex(@NotNull String text, int escSeqBeginInd) { - if (!text.regionMatches(escSeqBeginInd, CSI, 0, CSI.length())) { + private static int findEscSeqBeginIndex(@NotNull String text, int fromIndex) { + int ind = text.indexOf(CSI.charAt(0), fromIndex); + if (ind == -1) { return -1; } - int parameterEndInd = escSeqBeginInd + 2; + else if (ind == text.length() - 1) { + return encodeUnhandledSuffixLength(text, ind); + } + return text.charAt(ind + 1) == CSI.charAt(1) ? ind : -1; + } + + /** + * Returns end index of all consecutive escape sequences started at {@code firstEscSeqBeginInd}, or + * negative number if not found: -1 - (length of string suffix to keep in case of an incomplete last escape sequence). + */ + private static int findConsecutiveEscSequencesEndIndex(@NotNull String text, int firstEscSeqBeginInd) { + int escSeqBeginInd = firstEscSeqBeginInd; + int lastMatchedColorEscSeqEndInd = -1; + int escSeqEndInd; + while ((escSeqEndInd = findEscSeqEndIndex(text, escSeqBeginInd)) >= 0) { + if (text.charAt(escSeqEndInd) != 'm') { + // Handle non-color escape sequences separately + // ColoredOutputTypeRegistry expects only color escape sequences and in a single consecutive text chunk + return lastMatchedColorEscSeqEndInd > 0 ? lastMatchedColorEscSeqEndInd : escSeqEndInd; + } + escSeqBeginInd = escSeqEndInd + 1; + lastMatchedColorEscSeqEndInd = escSeqEndInd; + if (escSeqEndInd + 1 >= text.length()) { + return encodeUnhandledSuffixLength(text, firstEscSeqBeginInd); + } + if (text.charAt(escSeqEndInd + 1) != CSI.charAt(0)) { + break; + } + if (escSeqEndInd + 2 >= text.length()) { + return encodeUnhandledSuffixLength(text, firstEscSeqBeginInd); + } + if (text.charAt(escSeqEndInd + 2) != CSI.charAt(1)) { + break; + } + } + if (escSeqEndInd < -1) { + return encodeUnhandledSuffixLength(text, firstEscSeqBeginInd); + } + return lastMatchedColorEscSeqEndInd; + } + + private static int findEscSeqEndIndex(@NotNull String text, int escSeqBeginInd) { + int parameterEndInd = escSeqBeginInd + CSI.length(); while (parameterEndInd < text.length()) { char ch = text.charAt(parameterEndInd); if (Character.isDigit(ch) || ch == ';') { @@ -154,13 +218,21 @@ public class AnsiEscapeDecoder { break; } } - if (parameterEndInd < text.length()) { - char letter = text.charAt(parameterEndInd); - if (StringUtil.containsChar("ABCDEFGHJKSTfmisu", letter)) { - return parameterEndInd; - } + if (parameterEndInd == text.length()) { + return encodeUnhandledSuffixLength(text, escSeqBeginInd); } - return -1; + return StringUtil.containsChar("ABCDEFGHJKSTfmisu", text.charAt(parameterEndInd)) ? parameterEndInd : -1; + } + + private static int encodeUnhandledSuffixLength(@NotNull String text, int suffixStartInd) { + return -1 - (text.length() - suffixStartInd); + } + + private static int decodeUnhandledSuffixLength(int encodedUnhandledSuffixLength) { + if (encodedUnhandledSuffixLength >= -1) { + throw new AssertionError(); + } + return -encodedUnhandledSuffixLength - 1; } @Nullable diff --git a/platform/platform-tests/testSrc/com/intellij/execution/process/AnsiEscapeDecoderTest.java b/platform/platform-tests/testSrc/com/intellij/execution/process/AnsiEscapeDecoderTest.java index 5846480604bf..20a952fe1f7f 100644 --- a/platform/platform-tests/testSrc/com/intellij/execution/process/AnsiEscapeDecoderTest.java +++ b/platform/platform-tests/testSrc/com/intellij/execution/process/AnsiEscapeDecoderTest.java @@ -1,8 +1,6 @@ package com.intellij.execution.process; -import com.intellij.openapi.util.Key; import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.io.BufferExposingByteArrayInputStream; import com.intellij.testFramework.PlatformTestCase; import com.intellij.testFramework.PlatformTestUtil; import com.intellij.util.Consumer; @@ -10,101 +8,138 @@ import com.intellij.util.containers.ContainerUtil; import org.jetbrains.annotations.NotNull; import org.junit.Assert; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.Collection; +import java.io.*; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; public class AnsiEscapeDecoderTest extends PlatformTestCase { + private static final String STDOUT_KEY = ProcessOutputTypes.STDOUT.toString(); + private static final String STDERR_KEY = ProcessOutputTypes.STDERR.toString(); + public void testTextWithoutColors() throws Exception { - AnsiEscapeDecoder decoder = new AnsiEscapeDecoder(); - decoder.escapeText("", ProcessOutputTypes.STDOUT, createExpectedAcceptor( - Pair.create("", ProcessOutputTypes.STDOUT) - )); - decoder.escapeText("simple text", ProcessOutputTypes.STDOUT, createExpectedAcceptor( - Pair.create("simple text", ProcessOutputTypes.STDOUT) - )); + check(new ColoredText("")); + check(new ColoredText("simple text").addExpected("simple text", STDOUT_KEY)); } public void testSingleColoredChunk() throws Exception { - AnsiEscapeDecoder decoder = new AnsiEscapeDecoder(); - decoder.escapeText("Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n", ProcessOutputTypes.STDOUT, createExpectedAcceptor( - Pair.create("Chrome 35.0.1916 (Linux): Executed 0 of 1", ProcessOutputTypes.STDOUT), - Pair.create(" SUCCESS", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[32m")), - Pair.create(" (0 secs / 0 secs)\n", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[39m")) - )); + check(new ColoredText("Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n") + .addExpected("Chrome 35.0.1916 (Linux): Executed 0 of 1", STDOUT_KEY) + .addExpected(" SUCCESS", "\u001B[32m") + .addExpected(" (0 secs / 0 secs)\n", "\u001B[39m")); } public void testCompoundEscSeq() throws Exception { - AnsiEscapeDecoder decoder = new AnsiEscapeDecoder(); - decoder.escapeText("E\u001B[41m\u001B[37mE\u001B[0mE", ProcessOutputTypes.STDOUT, createExpectedAcceptor( - Pair.create("E", ProcessOutputTypes.STDOUT), - Pair.create("E", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[41;37m")), - Pair.create("E", ProcessOutputTypes.STDOUT) - )); + check(new ColoredText("E\u001B[41m\u001B[37mE\u001B[0mE") + .addExpected("E", STDOUT_KEY) + .addExpected("E", "\u001B[41;37m") + .addExpected("E", STDOUT_KEY)); } public void testOtherEscSeq() throws Exception { - AnsiEscapeDecoder decoder = new AnsiEscapeDecoder(); - decoder.escapeText("Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m", - ProcessOutputTypes.STDOUT, - createExpectedAcceptor( - Pair.create("Plain", ProcessOutputTypes.STDOUT), - Pair.create("Green", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[32m")), - Pair.create("Normal", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[39m")), - Pair.create("Red", ColoredOutputTypeRegistry.getInstance().getOutputKey("\u001B[31m")) - ) - ); + check(new ColoredText("Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m") + .addExpected("Plain", STDOUT_KEY) + .addExpected("Green", "\u001B[32m") + .addExpected("Normal", "\u001B[39m") + .addExpected("Red", "\u001B[31m")); } public void testBackspaceControlSequence() throws Exception { + check(false, ContainerUtil.newArrayList( + new ColoredText(" 10% 0/1 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 70% 1/1 build modules") + .addExpected(" 70% 1/1 build modules", STDOUT_KEY), + new ColoredText( + "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 40% 1/2 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 30% 1/3 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 25% 1/4 build modules") + .addExpected("\n 25% 1/4 build modules", STDOUT_KEY) + )); + } + + public void testIncompleteEscapeSequences() throws Exception { + check(true, ContainerUtil.newArrayList( + new ColoredText("\u001B"), + new ColoredText("[33m Hello\u001B[3").addExpected(" Hello", "\u001B[33m"), + new ColoredText("4m, Work!").addExpected(", Work!", "\u001B[34m") + )); + + check(true, ContainerUtil.newArrayList( + new ColoredText("\u001B[1;33m<\u001B[34mnamespace\u001B[1") + .addExpected("<", "\u001B[1;33m") + .addExpected("namespace", "\u001B[34m"), + new ColoredText( + ";33m:abcd\u001B[0m\u001B[1;33m>\u001B[0m0\u001B[1;33m\u001B[0m") + .addExpected(":abcd", "\u001B[1;33m") + .addExpected(">", "\u001B[0;1;33m") + .addExpected("0", "stdout") + .addExpected("", "\u001B[0;1;33m") + )); + } + + private static void check(@NotNull ColoredText text) { + check(true, Collections.singletonList(text)); + } + + private static void check(boolean testCharByCharProcessing, @NotNull List texts) { AnsiEscapeDecoder decoder = new AnsiEscapeDecoder(); - decoder.escapeText(" 10% 0/1 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 70% 1/1 build modules", - ProcessOutputTypes.STDERR, - createExpectedAcceptor( - Pair.create(" 70% 1/1 build modules", ProcessOutputTypes.STDERR) - ) - ); - decoder.escapeText("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 40% 1/2 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 30% 1/3 build modules\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b 25% 1/4 build modules", - ProcessOutputTypes.STDERR, - createExpectedAcceptor( - Pair.create("\n 25% 1/4 build modules", ProcessOutputTypes.STDERR) - ) - ); - } - - @NotNull - private static List> toListWithKeyName(@NotNull Collection> list) { - return ContainerUtil.map(list, pair -> Pair.create(pair.first, pair.second.toString())); - } - - @SafeVarargs - private static AnsiEscapeDecoder.ColoredChunksAcceptor createExpectedAcceptor(@NotNull final Pair... expected) { - return new AnsiEscapeDecoder.ColoredChunksAcceptor() { - @Override - public void coloredChunksAvailable(@NotNull List> chunks) { - List> expectedWithKeyName = toListWithKeyName(Arrays.asList(expected)); - List> actualWithKeyName = toListWithKeyName(chunks); - Assert.assertEquals(expectedWithKeyName, actualWithKeyName); - } - - @Override - public void coloredTextAvailable(@NotNull String text, @NotNull Key attributes) { - throw new RuntimeException(); // shouldn't be called - } + List> actualColoredChunks = ContainerUtil.newArrayList(); + //noinspection CodeBlock2Expr + AnsiEscapeDecoder.ColoredTextAcceptor acceptor = (text, attributes) -> { + actualColoredChunks.add(Pair.create(text, attributes.toString())); }; + // test stdout + for (ColoredText text : texts) { + decoder.escapeText(text.myRawText, ProcessOutputTypes.STDOUT, acceptor); + } + List> expectedColoredChunks = new ArrayList<>(); + for (ColoredText text : texts) { + expectedColoredChunks.addAll(text.myExpectedColoredChunks); + } + Assert.assertEquals(expectedColoredChunks, actualColoredChunks); + + if (testCharByCharProcessing) { + // test stdout char by char + actualColoredChunks.clear(); + decoder = new AnsiEscapeDecoder(); + for (ColoredText text : texts) { + for (int i = 0; i < text.myRawText.length(); i++) { + decoder.escapeText(String.valueOf(text.myRawText.charAt(i)), ProcessOutputTypes.STDOUT, acceptor); + } + } + expectedColoredChunks.clear(); + for (ColoredText text : texts) { + for (Pair chunk : text.myExpectedColoredChunks) { + String chunkText = chunk.first; + for (int i = 0; i < chunkText.length(); i++) { + expectedColoredChunks.add(Pair.create(String.valueOf(chunkText.charAt(i)), chunk.second)); + } + } + } + Assert.assertEquals(expectedColoredChunks, actualColoredChunks); + } + + // test stderr + actualColoredChunks.clear(); + decoder = new AnsiEscapeDecoder(); + for (ColoredText text : texts) { + decoder.escapeText(text.myRawText, ProcessOutputTypes.STDERR, acceptor); + } + expectedColoredChunks.clear(); + for (ColoredText text : texts) { + for (Pair chunk : text.myExpectedColoredChunks) { + expectedColoredChunks.add(Pair.create(chunk.first, STDERR_KEY)); + } + } + Assert.assertEquals(expectedColoredChunks, actualColoredChunks); } @NotNull public static Process createTestProcess() { // have to be synchronised because used from pooled thread ByteArrayOutputStream outputStream = new ByteArrayOutputStream(10000); - BufferExposingByteArrayInputStream inputStream = new BufferExposingByteArrayInputStream(new byte[0]); AtomicBoolean finished = new AtomicBoolean(); return new Process() { @Override @@ -114,17 +149,17 @@ public class AnsiEscapeDecoderTest extends PlatformTestCase { @Override public InputStream getInputStream() { - return inputStream; + return new ByteArrayInputStream(new byte[0]); } @Override public InputStream getErrorStream() { - return inputStream; + return new ByteArrayInputStream(new byte[0]); } @Override public int waitFor() { - while (!finished.get()); + while (!finished.get()) {} return 0; } @@ -143,11 +178,14 @@ public class AnsiEscapeDecoderTest extends PlatformTestCase { public void testPerformance() throws IOException { Process testProcess = createTestProcess(); + //noinspection CodeBlock2Expr withProcessHandlerFrom(testProcess, handler -> { - PlatformTestUtil.startPerformanceTest("ansi color", 15000, ()->{ - for (int i=0; i<2_000_000;i++) { - handler.notifyTextAvailable(i+"Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n", ProcessOutputTypes.STDOUT); - handler.notifyTextAvailable(i+"Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m\n", ProcessOutputTypes.SYSTEM); + PlatformTestUtil.startPerformanceTest("ansi color", 15000, () -> { + for (int i = 0; i < 2_000_000; i++) { + handler.notifyTextAvailable(i + "Chrome 35.0.1916 (Linux): Executed 0 of 1\u001B[32m SUCCESS\u001B[39m (0 secs / 0 secs)\n", + ProcessOutputTypes.STDOUT); + handler.notifyTextAvailable(i + "Plain\u001B[32mGreen\u001B[39mNormal\u001B[1A\u001B[2K\u001B[31mRed\u001B[39m\n", + ProcessOutputTypes.SYSTEM); } }).cpuBound().assertTiming(); }); @@ -168,4 +206,18 @@ public class AnsiEscapeDecoderTest extends PlatformTestCase { handler.waitFor(); } } + + private static class ColoredText { + private final String myRawText; + private final List> myExpectedColoredChunks = new ArrayList<>(); + + public ColoredText(@NotNull String rawText) { + myRawText = rawText; + } + + private ColoredText addExpected(@NotNull String text, @NotNull String colorKey) { + myExpectedColoredChunks.add(Pair.create(text, colorKey)); + return this; + } + } }