diff --git a/platform/core-impl/src/com/intellij/util/DocumentUtil.java b/platform/core-impl/src/com/intellij/util/DocumentUtil.java index 45256a181765..cd128e70de10 100644 --- a/platform/core-impl/src/com/intellij/util/DocumentUtil.java +++ b/platform/core-impl/src/com/intellij/util/DocumentUtil.java @@ -116,4 +116,26 @@ public final class DocumentUtil { public static boolean isAtLineEnd(int offset, @NotNull Document document) { return offset >= 0 && offset <= document.getTextLength() && offset == document.getLineEndOffset(document.getLineNumber(offset)); } + + public static int alignToCodePointBoundary(@NotNull Document document, int offset) { + return isInsideSurrogatePair(document, offset) ? offset - 1 : offset; + } + + public static boolean isSurrogatePair(@NotNull Document document, int offset) { + CharSequence text = document.getImmutableCharSequence(); + if (offset < 0 || (offset + 1) >= text.length()) return false; + return Character.isSurrogatePair(text.charAt(offset), text.charAt(offset + 1)); + } + + public static boolean isInsideSurrogatePair(@NotNull Document document, int offset) { + return isSurrogatePair(document, offset - 1); + } + + public static int getPreviousCodePointOffset(@NotNull Document document, int offset) { + return offset - (isSurrogatePair(document, offset - 2) ? 2 : 1); + } + + public static int getNextCodePointOffset(@NotNull Document document, int offset) { + return offset + (isSurrogatePair(document, offset) ? 2 : 1); + } } diff --git a/platform/lang-impl/src/com/intellij/codeInsight/editorActions/BackspaceHandler.java b/platform/lang-impl/src/com/intellij/codeInsight/editorActions/BackspaceHandler.java index ce48dedc9968..09b361bfc829 100644 --- a/platform/lang-impl/src/com/intellij/codeInsight/editorActions/BackspaceHandler.java +++ b/platform/lang-impl/src/com/intellij/codeInsight/editorActions/BackspaceHandler.java @@ -41,6 +41,7 @@ import com.intellij.psi.PsiDocumentManager; import com.intellij.psi.PsiFile; import com.intellij.psi.codeStyle.CodeStyleSettingsManager; import com.intellij.psi.util.PsiUtilBase; +import com.intellij.util.DocumentUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -73,10 +74,10 @@ public class BackspaceHandler extends EditorWriteActionHandler { if (editor.getSelectionModel().hasSelection()) return false; - int offset = editor.getCaretModel().getOffset() - 1; + int offset = DocumentUtil.getPreviousCodePointOffset(editor.getDocument(), editor.getCaretModel().getOffset()); if (offset < 0) return false; CharSequence chars = editor.getDocument().getCharsSequence(); - char c = chars.charAt(offset); + int c = Character.codePointAt(chars, offset); final Editor injectedEditor = TypedHandler.injectedEditorIfCharTypedIsSignificant(c, editor, file); final Editor originalEditor = editor; @@ -85,14 +86,14 @@ public class BackspaceHandler extends EditorWriteActionHandler { if (isOffsetInsideInjected(injectedEditor, injectedOffset)) { file = PsiDocumentManager.getInstance(project).getPsiFile(injectedEditor.getDocument()); editor = injectedEditor; - offset = injectedOffset - 1; + offset = DocumentUtil.getPreviousCodePointOffset(injectedEditor.getDocument(), injectedOffset); } } final BackspaceHandlerDelegate[] delegates = Extensions.getExtensions(BackspaceHandlerDelegate.EP_NAME); - if (!toWordStart) { + if (!toWordStart && Character.isBmpCodePoint(c)) { for(BackspaceHandlerDelegate delegate: delegates) { - delegate.beforeCharDeleted(c, file, editor); + delegate.beforeCharDeleted((char)c, file, editor); } } @@ -104,9 +105,9 @@ public class BackspaceHandler extends EditorWriteActionHandler { myOriginalHandler.execute(originalEditor, caret, dataContext); - if (!toWordStart) { + if (!toWordStart && Character.isBmpCodePoint(c)) { for(BackspaceHandlerDelegate delegate: delegates) { - if (delegate.charDeleted(c, file, editor)) { + if (delegate.charDeleted((char)c, file, editor)) { return true; } } @@ -117,7 +118,7 @@ public class BackspaceHandler extends EditorWriteActionHandler { chars = editor.getDocument().getCharsSequence(); if ((c == '(' || c == '[' || c == '{') && CodeInsightSettings.getInstance().AUTOINSERT_PAIR_BRACKET) { char c1 = chars.charAt(offset); - if (c1 != getRightChar(c)) return true; + if (c1 != getRightChar((char)c)) return true; HighlighterIterator iterator = ((EditorEx)editor).getHighlighter().createIterator(offset); BraceMatcher braceMatcher = BraceMatchingUtil.getBraceMatcher(fileType, iterator); diff --git a/platform/lang-impl/src/com/intellij/codeInsight/editorActions/TypedHandler.java b/platform/lang-impl/src/com/intellij/codeInsight/editorActions/TypedHandler.java index d971a9fa2401..f7dd8a142271 100644 --- a/platform/lang-impl/src/com/intellij/codeInsight/editorActions/TypedHandler.java +++ b/platform/lang-impl/src/com/intellij/codeInsight/editorActions/TypedHandler.java @@ -327,6 +327,11 @@ public class TypedHandler extends TypedActionHandlerBase { @NotNull public static Editor injectedEditorIfCharTypedIsSignificant(final char charTyped, @NotNull Editor editor, @NotNull PsiFile oldFile) { + return injectedEditorIfCharTypedIsSignificant((int)charTyped, editor, oldFile); + } + + @NotNull + public static Editor injectedEditorIfCharTypedIsSignificant(final int charTyped, @NotNull Editor editor, @NotNull PsiFile oldFile) { int offset = editor.getCaretModel().getOffset(); // even for uncommitted document try to retrieve injected fragment that has been there recently // we are assuming here that when user is (even furiously) typing, injected language would not change @@ -339,7 +344,8 @@ public class TypedHandler extends TypedActionHandlerBase { // IDEA-52375/WEB-9105 fix: last quote in editable fragment should be handled by outer language quote handler TextRange hostRange = documentWindow.getHostRange(offset); CharSequence sequence = editor.getDocument().getCharsSequence(); - if (sequence.length() > offset && charTyped != sequence.charAt(offset) || hostRange != null && hostRange.contains(offset)) { + if (sequence.length() > offset && charTyped != Character.codePointAt(sequence, offset) || + hostRange != null && hostRange.contains(offset)) { return injectedEditor; } } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/actions/BackspaceAction.java b/platform/platform-impl/src/com/intellij/openapi/editor/actions/BackspaceAction.java index 49ca3c09ec82..aac5e2d98532 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/actions/BackspaceAction.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/actions/BackspaceAction.java @@ -31,7 +31,7 @@ import com.intellij.openapi.editor.*; import com.intellij.openapi.editor.actionSystem.EditorWriteActionHandler; import com.intellij.openapi.editor.ex.util.EditorUIUtil; import com.intellij.openapi.editor.ex.util.EditorUtil; -import com.intellij.util.ui.MacUIUtil; +import com.intellij.util.DocumentUtil; import org.jetbrains.annotations.NotNull; public class BackspaceAction extends TextComponentEditorAction { @@ -89,7 +89,7 @@ public class BackspaceAction extends TextComponentEditorAction { editor.getCaretModel().moveToOffset(region.getStartOffset()); } else { - document.deleteString(offset - 1, offset); + document.deleteString(DocumentUtil.getPreviousCodePointOffset(document, offset), offset); } } } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/actions/DeleteAction.java b/platform/platform-impl/src/com/intellij/openapi/editor/actions/DeleteAction.java index 4bf02a4d31fa..7f7df040d1f5 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/actions/DeleteAction.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/actions/DeleteAction.java @@ -33,6 +33,7 @@ import com.intellij.openapi.editor.ex.util.EditorUIUtil; import com.intellij.openapi.editor.ex.util.EditorUtil; import com.intellij.openapi.ide.CopyPasteManager; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.DocumentUtil; import com.intellij.util.ui.MacUIUtil; public class DeleteAction extends EditorAction { @@ -104,7 +105,7 @@ public class DeleteAction extends EditorAction { editor.getCaretModel().moveToOffset(region.getStartOffset()); } else { - document.deleteString(offset, offset + 1); + document.deleteString(offset, DocumentUtil.getNextCodePointOffset(document, offset)); } return; } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/actions/EditorActionUtil.java b/platform/platform-impl/src/com/intellij/openapi/editor/actions/EditorActionUtil.java index dc4aecf0a135..7bc40ac6be51 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/actions/EditorActionUtil.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/actions/EditorActionUtil.java @@ -45,6 +45,7 @@ import com.intellij.psi.PsiDocumentManager; import com.intellij.psi.PsiFile; import com.intellij.psi.codeStyle.CodeStyleSettingsManager; import com.intellij.psi.tree.IElementType; +import com.intellij.util.DocumentUtil; import com.intellij.util.EditorPopupHandler; import com.intellij.util.text.CharArrayUtil; import org.jetbrains.annotations.NotNull; @@ -243,7 +244,11 @@ public class EditorActionUtil { * Finds out whether there's a boundary between two lexemes of different type at given offset. */ public static boolean isLexemeBoundary(@NotNull Editor editor, int offset) { - if (!(editor instanceof EditorEx) || offset <= 0 || offset >= editor.getDocument().getTextLength()) return false; + if (!(editor instanceof EditorEx) || + offset <= 0 || offset >= editor.getDocument().getTextLength() || + DocumentUtil.isInsideSurrogatePair(editor.getDocument(), offset)) { + return false; + } if (CharArrayUtil.isEmptyOrSpaces(editor.getDocument().getImmutableCharSequence(), offset - 1, offset + 1)) return false; EditorHighlighter highlighter = ((EditorEx)editor).getHighlighter(); HighlighterIterator it = highlighter.createIterator(offset); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretImpl.java index c3d95819de53..757b594ab7b7 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretImpl.java @@ -44,6 +44,7 @@ import com.intellij.util.ui.EmptyClipboardOwner; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; import java.awt.*; import java.awt.datatransfer.Clipboard; @@ -1090,8 +1091,8 @@ public class CaretImpl extends UserDataHolderBase implements Caret, Dumpable { final boolean updateSystemSelection) { myEditor.getCaretModel().doWithCaretMerging(() -> { - int startOffset = _startOffset; - int endOffset = _endOffset; + int startOffset = DocumentUtil.alignToCodePointBoundary(myEditor.getDocument(), _startOffset); + int endOffset = DocumentUtil.alignToCodePointBoundary(myEditor.getDocument(), _endOffset); myUnknownDirection = false; final Document doc = myEditor.getDocument(); @@ -1492,6 +1493,13 @@ public class CaretImpl extends UserDataHolderBase implements Caret, Dumpable { } } + @TestOnly + public void validateState() { + LOG.assertTrue(!DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), getOffset())); + LOG.assertTrue(!DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), getSelectionStart())); + LOG.assertTrue(!DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), getSelectionEnd())); + } + class PositionMarker extends RangeMarkerImpl { private PositionMarker(int offset) { super(myEditor.getDocument(), offset, offset, false); @@ -1518,6 +1526,11 @@ public class CaretImpl extends UserDataHolderBase implements Caret, Dumpable { setIntervalStart(afterInserted); setIntervalEnd(afterInserted); } + int offset = intervalStart(); + if (DocumentUtil.isInsideSurrogatePair(getDocument(), offset)) { + setIntervalStart(offset - 1); + setIntervalEnd(offset - 1); + } } else { setValid(true); @@ -1531,6 +1544,7 @@ public class CaretImpl extends UserDataHolderBase implements Caret, Dumpable { LOG.info(ex); } } + newOffset = DocumentUtil.alignToCodePointBoundary(getDocument(), newOffset); setIntervalStart(newOffset); setIntervalEnd(newOffset); } @@ -1579,6 +1593,12 @@ public class CaretImpl extends UserDataHolderBase implements Caret, Dumpable { @Override protected void changedUpdateImpl(@NotNull DocumentEvent e) { super.changedUpdateImpl(e); + if (isValid()) { + int startOffset = intervalStart(); + int endOffset = intervalEnd(); + if (DocumentUtil.isInsideSurrogatePair(getDocument(), startOffset)) setIntervalStart(startOffset - 1); + if (DocumentUtil.isInsideSurrogatePair(getDocument(), endOffset)) setIntervalStart(endOffset - 1); + } if (endVirtualOffset > 0 && isValid()) { Document document = e.getDocument(); int startAfter = intervalStart(); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretModelImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretModelImpl.java index 8446ab58f018..81221064428e 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretModelImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/CaretModelImpl.java @@ -40,6 +40,7 @@ import com.intellij.util.EventDispatcher; import com.intellij.util.containers.ContainerUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; @@ -575,6 +576,13 @@ public class CaretModelImpl implements CaretModel, PrioritizedDocumentListener, updateVisualPosition(); } + @TestOnly + public void validateState() { + for (CaretImpl caret : myCarets) { + caret.validateState(); + } + } + private static class VisualPositionComparator implements Comparator { private static final VisualPositionComparator INSTANCE = new VisualPositionComparator(); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/DefaultEditorTextRepresentationHelper.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/DefaultEditorTextRepresentationHelper.java index 04609be0fa06..a37dd3289233 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/DefaultEditorTextRepresentationHelper.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/DefaultEditorTextRepresentationHelper.java @@ -52,7 +52,7 @@ public class DefaultEditorTextRepresentationHelper implements EditorTextRepresen } @Override - public int charWidth(char c, int fontType) { + public int charWidth(int c, int fontType) { // Symbol width retrieval is detected to be a bottleneck, hence, we perform a caching here in assumption that every representation // helper is editor-bound and cache size is not too big. mySharedKey.fontType = fontType; @@ -61,7 +61,7 @@ public class DefaultEditorTextRepresentationHelper implements EditorTextRepresen return charWidth(c); } - private int charWidth(char c) { + private int charWidth(int c) { int result = mySymbolWidthCache.get(mySharedKey); if (result > 0) { return result; @@ -82,14 +82,14 @@ public class DefaultEditorTextRepresentationHelper implements EditorTextRepresen } private static class Key { - @JdkConstants.FontStyle private int fontType; - private char c; + @JdkConstants.FontStyle private int fontType; + private int c; private Key() { this(Font.PLAIN, ' '); } - Key(@JdkConstants.FontStyle int fontType, char c) { + Key(@JdkConstants.FontStyle int fontType, int c) { this.fontType = fontType; this.c = c; } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorImpl.java index 9c8ffbcf707a..2b2b7357be04 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorImpl.java @@ -4363,22 +4363,10 @@ public final class EditorImpl extends UserDataHolderBase implements EditorEx, Hi @TestOnly public void validateState() { myView.validateState(); - - if (myDocument.isInBulkUpdate()) return; - List softWraps = mySoftWrapModel.getRegisteredSoftWraps(); - int lastSoftWrapOffset = -1; - for (SoftWrap wrap : softWraps) { - int softWrapOffset = wrap.getStart(); - LOG.assertTrue(softWrapOffset > lastSoftWrapOffset, "Soft wraps are not ordered"); - LOG.assertTrue(softWrapOffset < myDocument.getTextLength(), "Soft wrap is after document's end"); - FoldRegion foldRegion = myFoldingModel.getCollapsedRegionAtOffset(softWrapOffset); - LOG.assertTrue(foldRegion == null || foldRegion.getStartOffset() == softWrapOffset, "Soft wrap is inside fold region"); - LOG.assertTrue(softWrapOffset != DocumentUtil.getLineEndOffset(softWrapOffset, myDocument) - || foldRegion != null, "Soft wrap before line break"); - LOG.assertTrue(softWrapOffset != DocumentUtil.getLineStartOffset(softWrapOffset, myDocument) || - myFoldingModel.isOffsetCollapsed(softWrapOffset - 1), "Soft wrap after line break"); - lastSoftWrapOffset = softWrapOffset; - } + mySoftWrapModel.validateState(); + myFoldingModel.validateState(); + myCaretModel.validateState(); + myInlayModel.validateState(); } private class MyScrollPane extends JBScrollPane { diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorTextRepresentationHelper.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorTextRepresentationHelper.java index 9e045c91d1d6..c99834b29b05 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorTextRepresentationHelper.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/EditorTextRepresentationHelper.java @@ -18,5 +18,5 @@ package com.intellij.openapi.editor.impl; import org.intellij.lang.annotations.JdkConstants; public interface EditorTextRepresentationHelper { - int charWidth(char c, @JdkConstants.FontStyle int fontType); + int charWidth(int c, @JdkConstants.FontStyle int fontType); } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldRegionImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldRegionImpl.java index f47aeac2352c..138661778f5c 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldRegionImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldRegionImpl.java @@ -29,6 +29,7 @@ import com.intellij.openapi.editor.FoldRegion; import com.intellij.openapi.editor.FoldingGroup; import com.intellij.openapi.editor.event.DocumentEvent; import com.intellij.openapi.editor.ex.DocumentEx; +import com.intellij.util.DocumentUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -140,6 +141,10 @@ class FoldRegionImpl extends RangeMarkerImpl implements FoldRegion { if (changeStart < oldEnd && changeEnd > oldStart) myDocumentRegionWasChanged = true; } super.changedUpdateImpl(e); + if (isValid() && (DocumentUtil.isInsideSurrogatePair(getDocument(), intervalStart()) || + DocumentUtil.isInsideSurrogatePair(getDocument(), intervalEnd()))) { + invalidate(e); + } } @Override diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldingModelImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldingModelImpl.java index dffa16a596cf..803348f220a5 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldingModelImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/FoldingModelImpl.java @@ -39,10 +39,12 @@ import com.intellij.openapi.editor.markup.TextAttributes; import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.ModificationTracker; +import com.intellij.util.DocumentUtil; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.containers.MultiMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; import java.awt.*; import java.util.Arrays; @@ -188,7 +190,11 @@ public class FoldingModelImpl implements FoldingModelEx, PrioritizedInternalDocu LOG.error("Fold regions must be added or removed inside batchFoldProcessing() only."); return false; } - + if (!region.isValid() || + DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), region.getStartOffset()) || + DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), region.getEndOffset())) { + return false; + } myFoldRegionsProcessed = true; if (myFoldTree.addRegion(region)) { final FoldingGroup group = region.getGroup(); @@ -606,6 +612,15 @@ public class FoldingModelImpl implements FoldingModelEx, PrioritizedInternalDocu return myExpansionCounter.get(); } + @TestOnly + public void validateState() { + for (FoldRegion region : getAllFoldRegions()) { + LOG.assertTrue (!region.isValid() || + !DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), region.getStartOffset()) && + !DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), region.getEndOffset())); + } + } + private static class SavedCaretPosition { private final LogicalPosition position; private final long docStamp; diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/FontInfo.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/FontInfo.java index 30329d71d815..a51886db9f80 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/FontInfo.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/FontInfo.java @@ -226,9 +226,9 @@ public class FontInfo { return myFont; } - public int charWidth(char c) { + public int charWidth(int codePoint) { final FontMetrics metrics = fontMetrics(); - return FontLayoutService.getInstance().charWidth(metrics, c); + return FontLayoutService.getInstance().charWidth(metrics, codePoint); } public FontMetrics fontMetrics() { diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayImpl.java index 69d5270dd124..4500ff520c78 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayImpl.java @@ -19,6 +19,7 @@ import com.intellij.openapi.editor.EditorCustomElementRenderer; import com.intellij.openapi.editor.Inlay; import com.intellij.openapi.editor.event.DocumentEvent; import com.intellij.openapi.util.Getter; +import com.intellij.util.DocumentUtil; import org.jetbrains.annotations.NotNull; class InlayImpl extends RangeMarkerImpl implements Inlay, Getter { @@ -60,6 +61,9 @@ class InlayImpl extends RangeMarkerImpl implements Inlay, Getter { setIntervalStart(newOffset); setIntervalEnd(newOffset); } + if (isValid() && DocumentUtil.isInsideSurrogatePair(getDocument(), intervalStart())) { + invalidate(e); + } } @Override diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayModelImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayModelImpl.java index 2e2dd64b1602..0ec3326aeaab 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayModelImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/InlayModelImpl.java @@ -17,6 +17,7 @@ package com.intellij.openapi.editor.impl; import com.intellij.openapi.Disposable; import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.editor.EditorCustomElementRenderer; import com.intellij.openapi.editor.Inlay; import com.intellij.openapi.editor.InlayModel; @@ -25,10 +26,12 @@ import com.intellij.openapi.editor.event.DocumentEvent; import com.intellij.openapi.editor.ex.DocumentEx; import com.intellij.openapi.editor.ex.PrioritizedDocumentListener; import com.intellij.openapi.util.Getter; +import com.intellij.util.DocumentUtil; import com.intellij.util.EventDispatcher; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; import java.awt.*; import java.util.ArrayList; @@ -37,6 +40,7 @@ import java.util.Comparator; import java.util.List; public class InlayModelImpl implements InlayModel, Disposable { + private static final Logger LOG = Logger.getInstance(InlayModelImpl.class); private static final Comparator INLAY_COMPARATOR = Comparator.comparingInt(Inlay::getOffset) .thenComparingInt(i -> ((InlayImpl)i).myOriginalOffset); @@ -109,6 +113,7 @@ public class InlayModelImpl implements InlayModel, Disposable { public Inlay addInlineElement(int offset, @NotNull EditorCustomElementRenderer renderer) { ApplicationManager.getApplication().assertIsDispatchThread(); DocumentEx document = myEditor.getDocument(); + if (DocumentUtil.isInsideSurrogatePair(document, offset)) return null; offset = Math.max(0, Math.min(document.getTextLength(), offset)); InlayImpl inlay = new InlayImpl(myEditor, offset, renderer); notifyAdded(inlay); @@ -175,4 +180,11 @@ public class InlayModelImpl implements InlayModel, Disposable { void notifyRemoved(InlayImpl inlay) { myDispatcher.getMulticaster().onRemoved(inlay); } + + @TestOnly + public void validateState() { + for (Inlay inlay : getInlineElementsInRange(0, myEditor.getDocument().getTextLength())) { + LOG.assertTrue(!DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), inlay.getOffset())); + } + } } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/SoftWrapModelImpl.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/SoftWrapModelImpl.java index 26f7e938b460..ae2eed33fb90 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/SoftWrapModelImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/SoftWrapModelImpl.java @@ -30,6 +30,7 @@ import com.intellij.openapi.editor.impl.softwrap.mapping.CachingSoftWrapDataMapp import com.intellij.openapi.editor.impl.softwrap.mapping.SoftWrapApplianceManager; import com.intellij.openapi.editor.impl.softwrap.mapping.SoftWrapAwareDocumentParsingListenerAdapter; import com.intellij.openapi.util.TextRange; +import com.intellij.util.DocumentUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; @@ -570,4 +571,26 @@ public class SoftWrapModelImpl extends InlayModel.SimpleAdapter public boolean isDirty() { return myUseSoftWraps && myDirty; } + + @TestOnly + public void validateState() { + Document document = myEditor.getDocument(); + if (myEditor.getDocument().isInBulkUpdate()) return; + FoldingModel foldingModel = myEditor.getFoldingModel(); + List softWraps = getRegisteredSoftWraps(); + int lastSoftWrapOffset = -1; + for (SoftWrap wrap : softWraps) { + int softWrapOffset = wrap.getStart(); + LOG.assertTrue(softWrapOffset > lastSoftWrapOffset, "Soft wraps are not ordered"); + LOG.assertTrue(softWrapOffset < document.getTextLength(), "Soft wrap is after document's end"); + FoldRegion foldRegion = foldingModel.getCollapsedRegionAtOffset(softWrapOffset); + LOG.assertTrue(foldRegion == null || foldRegion.getStartOffset() == softWrapOffset, "Soft wrap is inside fold region"); + LOG.assertTrue(softWrapOffset != DocumentUtil.getLineEndOffset(softWrapOffset, document) + || foldRegion != null, "Soft wrap before line break"); + LOG.assertTrue(softWrapOffset != DocumentUtil.getLineStartOffset(softWrapOffset, document) || + foldingModel.isOffsetCollapsed(softWrapOffset - 1), "Soft wrap after line break"); + LOG.assertTrue(!DocumentUtil.isInsideSurrogatePair(document, softWrapOffset), "Soft wrap inside a surrogate pair"); + lastSoftWrapOffset = softWrapOffset; + } + } } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/softwrap/mapping/SoftWrapApplianceManager.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/softwrap/mapping/SoftWrapApplianceManager.java index 1735201e1c39..96e7ecb26fe0 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/softwrap/mapping/SoftWrapApplianceManager.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/softwrap/mapping/SoftWrapApplianceManager.java @@ -36,6 +36,7 @@ import com.intellij.openapi.editor.impl.view.IterationState; import com.intellij.openapi.editor.markup.TextAttributes; import com.intellij.openapi.util.Segment; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.DocumentUtil; import org.intellij.lang.annotations.JdkConstants; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -402,9 +403,9 @@ public class SoftWrapApplianceManager implements Dumpable { return true; } - for (int j = softWrap.getStart(); j < myContext.tokenStartOffset; j++) { - char c = myContext.text.charAt(j); - myContext.onNonLineFeedSymbol(calculateNewX(c)); + while (myContext.currentPosition.offset < myContext.tokenStartOffset) { + int c = Character.codePointAt(myContext.text, myContext.currentPosition.offset); + myContext.onNonLineFeedSymbol(c, calculateNewX(c)); } myOffset2fontType.clear(); myContext.advance(foldRegion, placeholderWidthInPixels); @@ -430,8 +431,8 @@ public class SoftWrapApplianceManager implements Dumpable { LogMessageEx.error(LOG, "Cycled soft wraps recalculation detected", String.format( "Start recalculation offset: %d, visible area width: %d, calculation context: %s, editor info: %s", startOffset, myVisibleAreaWidth, myContext, myEditor.dumpState())); - for (int i = myContext.currentPosition.offset; i < myContext.tokenEndOffset; i++) { - char c = myContext.text.charAt(i); + while (myContext.currentPosition.offset < myContext.tokenEndOffset) { + int c = Character.codePointAt(myContext.text, myContext.currentPosition.offset); if (c == '\n') { myContext.onNewLine(); if (checkIsDoneAfterNewLine()) { @@ -454,7 +455,7 @@ public class SoftWrapApplianceManager implements Dumpable { } } - char c = myContext.text.charAt(offset); + int c = Character.codePointAt(myContext.text, offset); if (c == '\n') { myContext.onNewLine(); if (checkIsDoneAfterNewLine()) { @@ -478,7 +479,7 @@ public class SoftWrapApplianceManager implements Dumpable { } } else { - myContext.onNonLineFeedSymbol(metrics); + myContext.onNonLineFeedSymbol(c, metrics); } } return false; @@ -502,7 +503,7 @@ public class SoftWrapApplianceManager implements Dumpable { * @param c target symbol referenced by the given offset * @return 'x' coordinate of the right edge of document symbol referenced by the given offset */ - private int[] offsetToX(int offset, char c) { + private int[] offsetToX(int offset, int c) { if (myOffset2widthInPixels.end > offset && (myOffset2widthInPixels.anchor + myOffset2widthInPixels.end > offset)) { @@ -565,22 +566,22 @@ public class SoftWrapApplianceManager implements Dumpable { // We should process that accordingly. if (actualSoftWrapOffset > myContext.tokenEndOffset) { myContext.delayedSoftWrap = softWrap; - myContext.onNonLineFeedSymbol(myContext.text.charAt(offset)); + myContext.onNonLineFeedSymbol(Character.codePointAt(myContext.text, offset)); return false; } else if (actualSoftWrapOffset < offset) { if (revertedToFoldRegion == null) { - for (int j = offset - 1; j >= actualSoftWrapOffset; j--) { - int pixelsDiff = myOffset2widthInPixels.data[j - myOffset2widthInPixels.anchor]; - myContext.currentPosition.offset--; + while (myContext.currentPosition.offset > actualSoftWrapOffset) { + int prevOffset = Character.offsetByCodePoints(myContext.text, myContext.currentPosition.offset, -1); + int pixelsDiff = myOffset2widthInPixels.data[prevOffset - myOffset2widthInPixels.anchor]; + myContext.currentPosition.offset = prevOffset; myContext.currentPosition.x -= pixelsDiff; } } } else if (actualSoftWrapOffset > offset) { - myContext.onNonLineFeedSymbol(myContext.text.charAt(offset)); - for (int j = offset + 1; j < actualSoftWrapOffset; j++) { - myContext.onNonLineFeedSymbol(myContext.text.charAt(offset)); + while (myContext.currentPosition.offset < actualSoftWrapOffset) { + myContext.onNonLineFeedSymbol(Character.codePointAt(myContext.text, myContext.currentPosition.offset)); } } @@ -601,7 +602,7 @@ public class SoftWrapApplianceManager implements Dumpable { } // {newX, actualWidth} - private int[] calculateNewX(char c) { + private int[] calculateNewX(int c) { if (c == '\t') { int xStart = myContext.currentPosition.x + myContext.getInlaysPrefixWidth(); int xEnd = EditorUtil.nextTabStop(xStart, myEditor); @@ -656,9 +657,10 @@ public class SoftWrapApplianceManager implements Dumpable { softWrapOffset = myLineWrapPositionStrategy.calculateWrapPosition( document, myEditor.getProject(), minOffset, maxOffset, preferredOffset, true, true ); + if (DocumentUtil.isInsideSurrogatePair(document, softWrapOffset)) softWrapOffset--; } - if (softWrapOffset >= lineData.endLineOffset || softWrapOffset < 0 + if (softWrapOffset >= lineData.endLineOffset || softWrapOffset < 0 || softWrapOffset <= minOffset || (myCustomIndentUsedLastTime && softWrapOffset == lineData.nonWhiteSpaceSymbolOffset) || (softWrapOffset > preferredOffset && myContext.lastFoldStartPosition != null // Prefer to wrap on fold region backwards && myContext.lastFoldStartPosition.offset <= preferredOffset)) // to wrapping forwards. @@ -671,6 +673,7 @@ public class SoftWrapApplianceManager implements Dumpable { @NotNull private SoftWrapImpl registerSoftWrap(int offset, int spaceSize, LogicalLineData lineData) { + assert !DocumentUtil.isInsideSurrogatePair(myEditor.getDocument(), offset); int indentInColumns = 0; int indentInPixels = myPainter.getMinDrawingWidth(SoftWrapDrawingType.AFTER_SOFT_WRAP); if (myCustomIndentUsedLastTime) { @@ -737,7 +740,8 @@ public class SoftWrapApplianceManager implements Dumpable { } for (int i = preferredOffset - 1; i >= minOffsetToUse; i--) { char c = myContext.text.charAt(i); - if (c >= 0x2f00) { // Check this document for eastern languages unicode ranges - http://www.unicode.org/charts + // Check this document for eastern languages unicode ranges - http://www.unicode.org/charts + if (c >= 0x2f00 && Character.isBmpCodePoint(Character.codePointAt(myContext.text, i))) { return i + 1; } } @@ -1231,7 +1235,7 @@ public class SoftWrapApplianceManager implements Dumpable { lastFold = null; } - public void onNonLineFeedSymbol(char c) { + public void onNonLineFeedSymbol(int c) { int[] metrics; if (myOffset2widthInPixels.end > myContext.currentPosition.offset && (myOffset2widthInPixels.anchor + myOffset2widthInPixels.end > myContext.currentPosition.offset)) @@ -1243,11 +1247,11 @@ public class SoftWrapApplianceManager implements Dumpable { else { metrics = calculateNewX(c); } - onNonLineFeedSymbol(metrics); + onNonLineFeedSymbol(c, metrics); } @SuppressWarnings("MagicConstant") - public void onNonLineFeedSymbol(int[] metrics) { // {newX, actualWidth} + public void onNonLineFeedSymbol(int codePoint, int[] metrics) { // {newX, actualWidth} if (myOffset2widthInPixels.anchor <= 0) { myOffset2widthInPixels.anchor = currentPosition.offset; } @@ -1261,7 +1265,7 @@ public class SoftWrapApplianceManager implements Dumpable { myOffset2widthInPixels.end++; currentPosition.x = metrics[0]; - currentPosition.offset++; + currentPosition.offset += Character.isBmpCodePoint(codePoint) ? 1 : 2; fontType = myOffset2fontType.get(currentPosition.offset); } @@ -1317,15 +1321,14 @@ public class SoftWrapApplianceManager implements Dumpable { * [{@link #tokenStartOffset}; {@link #skipToLineEnd} is set to 'true' otherwise */ public boolean tryToShiftToNextLine() { - for (int i = currentPosition.offset; i < tokenEndOffset; i++) { - char c = text.charAt(i); - currentPosition.offset = i; + while (currentPosition.offset < tokenEndOffset) { + int c = Character.codePointAt(text, currentPosition.offset); if (c == '\n') { onNewLine(); // Assuming that offset is incremented during this method call return checkIsDoneAfterNewLine(); } else { - onNonLineFeedSymbol(offsetToX(i, c)); + onNonLineFeedSymbol(c, offsetToX(currentPosition.offset, c)); } } skipToLineEnd = true; diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/ComplexTextFragment.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/ComplexTextFragment.java index bd92d217ef5a..dcb47eca287d 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/ComplexTextFragment.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/ComplexTextFragment.java @@ -18,11 +18,13 @@ package com.intellij.openapi.editor.impl.view; import com.intellij.openapi.diagnostic.Logger; import com.intellij.util.BitUtil; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.awt.*; import java.awt.font.FontRenderContext; import java.awt.font.GlyphVector; import java.awt.geom.Rectangle2D; +import java.util.Arrays; /** * GlyphVector-based text fragment. Used for non-Latin text or when ligatures are enabled @@ -33,6 +35,10 @@ class ComplexTextFragment extends TextFragment { @NotNull private final GlyphVector myGlyphVector; + @Nullable + private final short[] myCodePoint2Offset; // Start offset of each Unicode code point in the fragment + // (null if each code point takes one char). + // We expect no more than 1025 chars in a fragment, so 'short' should be enough. ComplexTextFragment(@NotNull char[] lineChars, int start, int end, boolean isRtl, @NotNull Font font, @NotNull FontRenderContext fontRenderContext) { @@ -83,6 +89,22 @@ class ComplexTextFragment extends TextFragment { setCharPosition(j, prevX + (lastX - prevX) * (j - lastCharIndex + 1) / (numChars - lastCharIndex), isRtl, numChars); } } + int codePointCount = Character.codePointCount(lineChars, start, end - start); + if (codePointCount == numChars) { + myCodePoint2Offset = null; + } + else { + myCodePoint2Offset = new short[codePointCount]; + int offset = 0; + for (int i = 0; i < codePointCount; i++) { + myCodePoint2Offset[i] = (short)(offset++); + if (offset < numChars && + Character.isHighSurrogate(lineChars[start + offset - 1]) && + Character.isLowSurrogate(lineChars[start + offset])) { + offset++; + } + } + } } private void setCharPosition(int logicalCharIndex, float x, boolean isRtl, int numChars) { @@ -96,6 +118,15 @@ class ComplexTextFragment extends TextFragment { return BitUtil.isSet(myGlyphVector.getLayoutFlags(), GlyphVector.FLAG_RUN_RTL); } + @Override + int offsetToLogicalColumn(int offset) { + if (myCodePoint2Offset == null) return offset; + if (offset == getLength()) return myCodePoint2Offset.length; + int i = Arrays.binarySearch(myCodePoint2Offset, (short)offset); + assert i >= 0; + return i; + } + // Drawing a portion of glyph vector using clipping might be not very effective // (we still pass all glyphs to the rendering code, and filtering by clipping might occur late in the processing, // on OS X larger number of glyphs passed for processing is known to slow down rendering significantly). @@ -154,6 +185,48 @@ class ComplexTextFragment extends TextFragment { } } + private int getCodePointCount() { + return myCodePoint2Offset == null ? myCharPositions.length : myCodePoint2Offset.length; + } + + private int visualColumnToVisualOffset(int column) { + if (myCodePoint2Offset == null) return column; + if (column <= 0) return 0; + if (column >= myCodePoint2Offset.length) return getLength(); + return isRtl() ? getLength() - myCodePoint2Offset[myCodePoint2Offset.length - column] : myCodePoint2Offset[column]; + } + + @Override + public int getLogicalColumnCount(int startColumn) { + return getCodePointCount(); + } + + @Override + public int getVisualColumnCount(float startX) { + return getCodePointCount(); + } + + @Override + public int[] xToVisualColumn(float startX, float x) { + float relX = x - startX; + float prevPos = 0; + int columnCount = getCodePointCount(); + for (int i = 0; i < columnCount; i++) { + int visualOffset = visualColumnToVisualOffset(i); + float newPos = myCharPositions[visualOffset]; + if (relX < (newPos + prevPos) / 2) { + return new int[] {i, relX <= prevPos ? 0 : 1}; + } + prevPos = newPos; + } + return new int[] {columnCount, relX <= myCharPositions[myCharPositions.length - 1] ? 0 : 1}; + } + + @Override + public float visualColumnToX(float startX, int column) { + return startX + getX(visualColumnToVisualOffset(column)); + } + public static void flushDrawingCache(Graphics2D g) { if (lastFragment != null) { g.setColor(lastColor); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorCoordinateMapper.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorCoordinateMapper.java index 4a58895ea0dc..ee2f8eaea63f 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorCoordinateMapper.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorCoordinateMapper.java @@ -23,6 +23,7 @@ import com.intellij.openapi.editor.impl.EditorImpl; import com.intellij.openapi.editor.impl.FoldingModelImpl; import com.intellij.openapi.editor.impl.SoftWrapModelImpl; import com.intellij.openapi.editor.impl.softwrap.SoftWrapDrawingType; +import com.intellij.util.DocumentUtil; import org.jetbrains.annotations.NotNull; import java.awt.*; @@ -186,6 +187,7 @@ class EditorCoordinateMapper { return 0; } offset = Math.min(offset, textLength); + offset = DocumentUtil.alignToCodePointBoundary(myDocument, offset); FoldRegion outermostCollapsed = myFoldingModel.getCollapsedRegionAtOffset(offset); if (outermostCollapsed != null && offset > outermostCollapsed.getStartOffset()) { @@ -224,6 +226,7 @@ class EditorCoordinateMapper { private int visualLineStartOffset(int offset, boolean leanForward) { EditorImpl editor = myView.getEditor(); + offset = DocumentUtil.alignToCodePointBoundary(myDocument, offset); int result = EditorUtil.getNotFoldedLineStartOffset(editor, offset); SoftWrapModelImpl softWrapModel = editor.getSoftWrapModel(); @@ -331,6 +334,7 @@ class EditorCoordinateMapper { @NotNull Point offsetToXY(int offset, boolean leanTowardsLargerOffsets, boolean beforeSoftWrap) { offset = Math.max(0, Math.min(myDocument.getTextLength(), offset)); + offset = DocumentUtil.alignToCodePointBoundary(myDocument, offset); int logicalLine = myDocument.getLineNumber(offset); int visualLine = offsetToVisualLine(offset, beforeSoftWrap); int visualLineStartOffset = visualLineToOffset(visualLine); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorPainter.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorPainter.java index 952e79c5a703..1be30dd71c32 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorPainter.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorPainter.java @@ -34,6 +34,7 @@ import com.intellij.ui.ColorUtil; import com.intellij.ui.paint.EffectPainter; import com.intellij.ui.Gray; import com.intellij.ui.JBColor; +import com.intellij.util.DocumentUtil; import com.intellij.util.text.CharArrayUtil; import com.intellij.util.ui.JBUI; import com.intellij.util.ui.UIUtil; @@ -611,6 +612,8 @@ class EditorPainter implements TextDrawingCallback { } private void paintBorderEffect(Graphics2D g, ClipDetector clipDetector, int startOffset, int endOffset, TextAttributes attributes) { + startOffset = DocumentUtil.alignToCodePointBoundary(myDocument, startOffset); + endOffset = DocumentUtil.alignToCodePointBoundary(myDocument, endOffset); if (!clipDetector.rangeCanBeVisible(startOffset, endOffset)) return; int startLine = myDocument.getLineNumber(startOffset); int endLine = myDocument.getLineNumber(endOffset); @@ -851,6 +854,7 @@ class EditorPainter implements TextDrawingCallback { int startX = Math.max(minX, isRtl ? x - width : x); g.fillRect(startX, y, width, nominalLineHeight - 1); if (myDocument.getTextLength() > 0 && caret != null) { + int charCount = DocumentUtil.isSurrogatePair(myDocument, caret.getOffset()) ? 2 : 1; int targetVisualColumn = caret.getVisualPosition().column; for (VisualLineFragmentsIterator.Fragment fragment : VisualLineFragmentsIterator.create(myView, caret.getVisualLineStart(), @@ -863,8 +867,8 @@ class EditorPainter implements TextDrawingCallback { endVisualColumn == targetVisualColumn && isRtl) { g.setColor(ColorUtil.isDark(caretColor) ? CARET_LIGHT : CARET_DARK); fragment.draw(g, startX, y + topOverhang + myView.getAscent(), - targetVisualColumn - startVisualColumn - (isRtl ? 1 : 0), - targetVisualColumn - startVisualColumn + (isRtl ? 0 : 1)); + targetVisualColumn - startVisualColumn - (isRtl ? charCount : 0), + targetVisualColumn - startVisualColumn + (isRtl ? 0 : charCount)); break; } } @@ -907,7 +911,8 @@ class EditorPainter implements TextDrawingCallback { SoftWrap softWrap = myEditor.getSoftWrapModel().getSoftWrap(offset); if (softWrap != null) { prevEndOffset = offset; - it = new IterationState(myEditor, offset == 0 ? 0 : offset - 1, visualLineEndOffset, caretData, false, false, false, false); + it = new IterationState(myEditor, offset == 0 ? 0 : DocumentUtil.getPreviousCodePointOffset(myDocument, offset), visualLineEndOffset, + caretData, false, false, false, false); if (it.getEndOffset() <= offset) { it.advance(); } @@ -970,8 +975,9 @@ class EditorPainter implements TextDrawingCallback { maxColumn = fragment.getEndVisualColumn(); } if (it == null || it.getEndOffset() != visualLineEndOffset) { - it = new IterationState(myEditor, visualLineEndOffset == offset ? visualLineEndOffset : visualLineEndOffset - 1, visualLineEndOffset, - caretData, false, false, false, false); + it = new IterationState(myEditor, visualLineEndOffset == offset ? visualLineEndOffset + : DocumentUtil.getPreviousCodePointOffset(myDocument, visualLineEndOffset), + visualLineEndOffset, caretData, false, false, false, false); } if (!it.atEnd()) { it.advance(); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/FontLayoutService.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/FontLayoutService.java index 1be406c796b4..2b91a4d4f2a5 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/FontLayoutService.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/FontLayoutService.java @@ -37,10 +37,12 @@ public abstract class FontLayoutService { @NotNull public abstract GlyphVector layoutGlyphVector(@NotNull Font font, @NotNull FontRenderContext fontRenderContext, - @NotNull char[] chars, int start, int end, boolean isRtl); - + @NotNull char[] chars, int start, int end, boolean isRtl); + public abstract int charWidth(@NotNull FontMetrics fontMetrics, char c); - + + public abstract int charWidth(@NotNull FontMetrics fontMetrics, int codePoint); + public abstract int getHeight(@NotNull FontMetrics fontMetrics); public abstract int getAscent(@NotNull FontMetrics fontMetrics); @@ -67,6 +69,11 @@ public abstract class FontLayoutService { return fontMetrics.charWidth(c); } + @Override + public int charWidth(@NotNull FontMetrics fontMetrics, int codePoint) { + return fontMetrics.charWidth(codePoint); + } + @Override public int getHeight(@NotNull FontMetrics fontMetrics) { return fontMetrics.getHeight(); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/IterationState.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/IterationState.java index 46a0cd5b567e..e2bae917c390 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/IterationState.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/IterationState.java @@ -29,6 +29,7 @@ import com.intellij.openapi.editor.markup.TextAttributes; import com.intellij.openapi.util.Comparing; import com.intellij.util.ArrayUtilRt; import com.intellij.util.CommonProcessors; +import com.intellij.util.DocumentUtil; import com.intellij.util.ObjectUtils; import com.intellij.util.containers.ContainerUtil; import org.jetbrains.annotations.NotNull; @@ -129,8 +130,11 @@ public class IterationState { boolean useOnlyFontOrForegroundAffectingHighlighters, boolean useFoldRegions, boolean iterateBackwards) { ApplicationManager.getApplication().assertReadAccessAllowed(); myDocument = editor.getDocument(); - myStartOffset = start; + assert !DocumentUtil.isInsideSurrogatePair(myDocument, start); + assert !DocumentUtil.isInsideSurrogatePair(myDocument, end); + + myStartOffset = start; myEnd = end; myEditor = editor; myUseOnlyFullLineHighlighters = useOnlyFullLineHighlighters; @@ -227,8 +231,8 @@ public class IterationState { private void advance() { if (myNextHighlighter != null) { if (myReverseIteration ? - myNextHighlighter.getAffectedAreaEndOffset() < myStartOffset : - myNextHighlighter.getAffectedAreaStartOffset() > myStartOffset) { + getAlignedEndOffset(myNextHighlighter) < myStartOffset : + getAlignedStartOffset(myNextHighlighter) > myStartOffset) { return; } @@ -240,8 +244,8 @@ public class IterationState { RangeHighlighterEx highlighter = highlighters[i++]; if (!skipHighlighter(highlighter)) { if (myReverseIteration ? - highlighter.getAffectedAreaEndOffset() < myStartOffset : - highlighter.getAffectedAreaStartOffset() > myStartOffset) { + getAlignedEndOffset(highlighter) < myStartOffset : + getAlignedStartOffset(highlighter) > myStartOffset) { myNextHighlighter = highlighter; break; } @@ -254,7 +258,7 @@ public class IterationState { private int getMinSegmentHighlighterEnd() { if (myNextHighlighter != null) { - return myReverseIteration ? myNextHighlighter.getAffectedAreaEndOffset(): myNextHighlighter.getAffectedAreaStartOffset(); + return myReverseIteration ? getAlignedEndOffset(myNextHighlighter) : getAlignedStartOffset(myNextHighlighter); } return myReverseIteration ? Integer.MIN_VALUE : Integer.MAX_VALUE; } @@ -293,6 +297,8 @@ public class IterationState { myNextIsFoldRegion = myEndOffset == foldRangesEnd && myEndOffset < myEnd; } + assert !DocumentUtil.isInsideSurrogatePair(myDocument, myEndOffset); + reinit(); } @@ -307,7 +313,7 @@ public class IterationState { return myEnd; } while (!myHighlighterIterator.atEnd()) { - int end = myReverseIteration ? myHighlighterIterator.getStart() : myHighlighterIterator.getEnd(); + int end = alignOffset(myReverseIteration ? myHighlighterIterator.getStart() : myHighlighterIterator.getEnd()); if (myReverseIteration ? end < start : end > start) { return end; } @@ -355,7 +361,7 @@ public class IterationState { //noinspection ForLoopReplaceableByForEach for (int i = 0; i < blocks.size(); i++) { RangeMarker block = blocks.get(i); - int nearestValue = getNearestValueAhead(start, block.getStartOffset(), block.getEndOffset()); + int nearestValue = getNearestValueAhead(start, alignOffset(block.getStartOffset()), alignOffset(block.getEndOffset())); result = myReverseIteration ? Math.max(result, nearestValue) : Math.min(result, nearestValue); } return result; @@ -396,10 +402,10 @@ public class IterationState { for (int i = myCurrentHighlighters.size() - 1; i >= 0; i--) { RangeHighlighterEx highlighter = myCurrentHighlighters.get(i); if (myReverseIteration ? - highlighter.getAffectedAreaStartOffset() >= myStartOffset : + getAlignedStartOffset(highlighter) >= myStartOffset : fileEnd && highlighter.getTargetArea() == HighlighterTargetArea.LINES_IN_RANGE ? - highlighter.getAffectedAreaEndOffset() < myStartOffset : - highlighter.getAffectedAreaEndOffset() <= myStartOffset) { + getAlignedEndOffset(highlighter) < myStartOffset : + getAlignedEndOffset(highlighter) <= myStartOffset) { myCurrentHighlighters.remove(i); } } @@ -460,14 +466,10 @@ public class IterationState { for (int i = 0; i < myCurrentHighlighters.size(); i++) { RangeHighlighterEx highlighter = myCurrentHighlighters.get(i); if (myReverseIteration) { - if (highlighter.getAffectedAreaStartOffset() > end) { - end = highlighter.getAffectedAreaStartOffset(); - } + end = Math.max(end, getAlignedStartOffset(highlighter)); } else { - if (highlighter.getAffectedAreaEndOffset() < end) { - end = highlighter.getAffectedAreaEndOffset(); - } + end = Math.min(end, getAlignedEndOffset(highlighter)); } } @@ -659,6 +661,18 @@ public class IterationState { myCaretRowAttributes.getBackgroundColor() : myDefaultBackground; } + private int alignOffset(int offset) { + return DocumentUtil.alignToCodePointBoundary(myDocument, offset); + } + + private int getAlignedStartOffset(RangeHighlighterEx highlighter) { + return alignOffset(highlighter.getAffectedAreaStartOffset()); + } + + private int getAlignedEndOffset(RangeHighlighterEx highlighter) { + return alignOffset(highlighter.getAffectedAreaEndOffset()); + } + private static class LayerComparator implements Comparator { private static final LayerComparator INSTANCE = new LayerComparator(); @Override diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineFragment.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineFragment.java index cb24dab79173..e00929d7b5b1 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineFragment.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineFragment.java @@ -21,13 +21,9 @@ import java.awt.*; /** * A building block of text line layout, that knows how to draw itself, and convert between offset, column and x coordinate within itself. - *

- * Existing implementations: {@link TextFragment} subclasses and {@link TabFragment}. - * Adding additional ones should be done with care as the code using them relies on specific properties of these implementations - * (e.g. fragments that have column count different from their length, like {@link TabFragment} shouldn't be reordered visually - * with respect to logically surrounding fragments and always belong to LTR runs). */ interface LineFragment { + // offset-based int getLength(); int getLogicalColumnCount(int startColumn); @@ -52,8 +48,8 @@ interface LineFragment { // offsets are visual float offsetToX(float startX, int startOffset, int offset); - // columns are visual - void draw(Graphics2D g, float x, float y, int startColumn, int endColumn); + // offsets are visual + void draw(Graphics2D g, float x, float y, int startOffset, int endOffset); // offsets are logical @NotNull diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineLayout.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineLayout.java index 29020c4242ad..d9ff57e01bb0 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineLayout.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineLayout.java @@ -30,6 +30,7 @@ import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.StringEscapesTokenTypes; import com.intellij.psi.tree.IElementType; import com.intellij.util.BitUtil; +import com.intellij.util.DocumentUtil; import com.intellij.util.text.CharArrayUtil; import org.intellij.lang.annotations.JdkConstants; import org.jetbrains.annotations.NotNull; @@ -58,7 +59,7 @@ abstract class LineLayout { @NotNull static LineLayout create(@NotNull EditorView view, int line, boolean skipBidiLayout) { List runs = createFragments(view, line, skipBidiLayout); - return createLayout(runs); + return createLayout(view, runs, null, line); } /** @@ -67,11 +68,11 @@ abstract class LineLayout { @NotNull static LineLayout create(@NotNull EditorView view, @NotNull CharSequence text, @JdkConstants.FontStyle int fontStyle) { List runs = createFragments(view, text, fontStyle); - LineLayout delegate = createLayout(runs); + LineLayout delegate = createLayout(view, runs, text, 0); return new WithSize(delegate); } - - private static LineLayout createLayout(@NotNull List runs) { + + private static LineLayout createLayout(@NotNull EditorView view, @NotNull List runs, @Nullable CharSequence text, int line) { if (runs.isEmpty()) return new SingleChunk(null); if (runs.size() == 1) { BidiRun run = runs.get(0); @@ -80,7 +81,20 @@ abstract class LineLayout { return new SingleChunk(chunk); } } - return new MultiChunk(runs.toArray(new BidiRun[runs.size()])); + BidiRun[] runArray = new BidiRun[runs.size()]; + int prevColumn = 0; + for (int i = 0; i < runs.size(); i++) { + BidiRun run = runs.get(i); + assert i == 0 || run.startOffset == runs.get(i - 1).endOffset; + int startColumn = prevColumn; + int endColumn = text == null + ? view.getLogicalPositionCache().offsetToLogicalColumn(line, run.endOffset) + : LogicalPositionCache.calcColumn(text, run.startOffset, prevColumn, run.endOffset, view.getTabSize()); + run.visualStartLogicalColumn = run.isRtl() ? endColumn : startColumn; + prevColumn = endColumn; + runArray[i] = run; + } + return new MultiChunk(runArray); } // runs are supposed to be in logical order initially @@ -116,7 +130,7 @@ abstract class LineLayout { char[] chars = CharArrayUtil.fromSequence(text); List runs = createRuns(view, chars, -1); for (BidiRun run : runs) { - for (Chunk chunk : run.getChunks()) { + for (Chunk chunk : run.getChunks(text, 0)) { chunk.fragments = new ArrayList<>(); addFragments(run, chunk, chars, chunk.startOffset, chunk.endOffset, fontStyle, fontPreferences, fontRenderContext, null); } @@ -133,39 +147,59 @@ abstract class LineLayout { int flags = view.getBidiFlags(); if (startOffsetInEditor >= 0) { // running bidi algorithm separately for text fragments corresponding to different lexer tokens - int lastOffset = startOffsetInEditor; + int relLastOffset = 0; IElementType lastToken = null; HighlighterIterator iterator = view.getEditor().getHighlighter().createIterator(startOffsetInEditor); - int endOffsetInEditor = startOffsetInEditor + textLength; - while (!iterator.atEnd() && iterator.getStart() < endOffsetInEditor) { - int relStartOffset = Math.max(0, iterator.getStart() - startOffsetInEditor); + while (!iterator.atEnd() && (iterator.getStart() - startOffsetInEditor) < textLength) { + int iteratorRelStart = alignToCodePointBoundary(text, iterator.getStart() - startOffsetInEditor); + int iteratorRelEnd = alignToCodePointBoundary(text, iterator.getEnd() - startOffsetInEditor); IElementType currentToken = iterator.getTokenType(); + int relStartOffset = Math.max(0, iteratorRelStart); String lcPrefix = getLineCommentPrefix(currentToken); - if (!StringUtil.isEmpty(lcPrefix) && lcPrefix.length() <= (iterator.getEnd() - iterator.getStart()) && - CharArrayUtil.regionMatches(text, relStartOffset, relStartOffset + lcPrefix.length(), lcPrefix)) { - addRuns(runs, text, lastOffset - startOffsetInEditor, relStartOffset, flags); - int textStartOffset = Math.min(textLength, Math.min(iterator.getEnd() - startOffsetInEditor, + // for line comments we process prefix and following text separately + if (!StringUtil.isEmpty(lcPrefix) && lcPrefix.length() <= (iteratorRelEnd - iteratorRelStart) && + CharArrayUtil.regionMatches(text, relStartOffset, relStartOffset + lcPrefix.length(), lcPrefix) && + !isInsideSurrogatePair(text, relStartOffset + lcPrefix.length())) { + addRuns(runs, text, relLastOffset, relStartOffset, flags); + int textStartOffset = Math.min(textLength, Math.min(iteratorRelEnd, CharArrayUtil.shiftForward(text, relStartOffset + lcPrefix.length(), " \t"))); - lastOffset = Math.min(iterator.getEnd(), endOffsetInEditor); + relLastOffset = Math.min(iteratorRelEnd, textLength); lastToken = null; addRuns(runs, text, relStartOffset, textStartOffset, flags); - addRuns(runs, text, textStartOffset, lastOffset - startOffsetInEditor, flags); + addRuns(runs, text, textStartOffset, relLastOffset, flags); } else if (distinctTokens(lastToken, currentToken)) { - addRuns(runs, text, lastOffset - startOffsetInEditor, relStartOffset, flags); + addRuns(runs, text, relLastOffset, relStartOffset, flags); lastToken = currentToken; - lastOffset = relStartOffset + startOffsetInEditor; + relLastOffset = relStartOffset; } iterator.advance(); } - addRuns(runs, text, lastOffset - startOffsetInEditor, endOffsetInEditor - startOffsetInEditor, flags); + addRuns(runs, text, relLastOffset, textLength, flags); } else { addRuns(runs, text, 0, textLength, flags); } + for (BidiRun run : runs) { + assert !isInsideSurrogatePair(text, run.startOffset); + assert !isInsideSurrogatePair(text, run.endOffset); + } return runs; } + private static boolean isInsideSurrogatePair(char[] text, int offset) { + return offset > 0 && offset < text.length && Character.isHighSurrogate(text[offset - 1]) && Character.isLowSurrogate(text[offset]); + } + + private static int alignToCodePointBoundary(char[] text, int offset) { + return isInsideSurrogatePair(text, offset) ? offset - 1 : offset; + } + + private static int alignToCodePointBoundary(CharSequence text, int offset) { + return offset > 0 && offset < text.length() && + Character.isHighSurrogate(text.charAt(offset - 1)) && Character.isLowSurrogate(text.charAt(offset)) ? offset - 1 : offset; + } + private static String getLineCommentPrefix(IElementType token) { if (token == null) return null; Commenter commenter = LanguageCommenters.INSTANCE.forLanguage(token.getLanguage()); @@ -272,7 +306,7 @@ abstract class LineLayout { return new Iterable() { @Override public Iterator iterator() { - return new VisualOrderIterator(null, 0, startX, 0, 0, 0, getRunsInVisualOrder()); + return new VisualOrderIterator(null, 0, startX, 0, 0, getRunsInVisualOrder()); } }; } @@ -289,6 +323,11 @@ abstract class LineLayout { int endOffset, @Nullable Runnable quickEvaluationListener) { assert startOffset <= endOffset; + Document document = view.getEditor().getDocument(); + int lineStartOffset = document.getLineStartOffset(line); + assert !DocumentUtil.isInsideSurrogatePair(document, lineStartOffset + startOffset); + assert !DocumentUtil.isInsideSurrogatePair(document, lineStartOffset + endOffset); + final BidiRun[] runs; if (startOffset == endOffset) { runs = BidiRun.EMPTY_ARRAY; @@ -305,8 +344,7 @@ abstract class LineLayout { reorderRunsVisually(runs); } } - final int startLogicalColumn = view.getLogicalPositionCache().offsetToLogicalColumn(line, startOffset); - return new VisualOrderIterator(view, line, startX, startVisualColumn, startLogicalColumn, startOffset, runs); + return new VisualOrderIterator(view, line, startX, startVisualColumn, startOffset, runs); } abstract Stream getChunksInLogicalOrder(); @@ -505,6 +543,7 @@ abstract class LineLayout { private final byte level; private final int startOffset; private final int endOffset; + private int visualStartLogicalColumn; private Chunk[] chunks; // in logical order private BidiRun(int length) { @@ -521,14 +560,15 @@ abstract class LineLayout { return BitUtil.isSet(level, 1); } - private Chunk[] getChunks() { + private Chunk[] getChunks(CharSequence text, int startOffsetInText) { if (chunks == null) { int chunkCount = getChunkCount(); chunks = new Chunk[chunkCount]; for (int i = 0; i < chunkCount; i++) { int from = startOffset + i * CHUNK_CHARACTERS; int to = (i == chunkCount - 1) ? endOffset : from + CHUNK_CHARACTERS; - Chunk chunk = new Chunk(from, to); + Chunk chunk = new Chunk(alignToCodePointBoundary(text, from + startOffsetInText) - startOffsetInText, + alignToCodePointBoundary(text, to + startOffsetInText) - startOffsetInText); chunks[i] = chunk; } } @@ -547,12 +587,15 @@ abstract class LineLayout { int end = Math.min(endOffset, targetEndOffset); BidiRun subRun = new BidiRun(level, start, end); List subChunks = new ArrayList<>(); - for (Chunk chunk : getChunks()) { + Document document = view.getEditor().getDocument(); + for (Chunk chunk : getChunks(document.getImmutableCharSequence(), document.getLineStartOffset(line))) { if (chunk.endOffset <= start) continue; if (chunk.startOffset >= end) break; subChunks.add(chunk.subChunk(view, this, line, start, end, quickEvaluationListener)); } subRun.chunks = subChunks.toArray(new Chunk[subChunks.size()]); + subRun.visualStartLogicalColumn = (subRun.isRtl() ? end == endOffset : start == startOffset) ? visualStartLogicalColumn : + view.getLogicalPositionCache().offsetToLogicalColumn(line, subRun.isRtl() ? end : start); return subRun; } } @@ -663,7 +706,9 @@ abstract class LineLayout { private static class VisualOrderIterator implements Iterator { private final EditorView myView; + private final CharSequence myText; private final int myLine; + private final int myLineStartOffset; private final BidiRun[] myRuns; private int myRunIndex = 0; private int myChunkIndex = 0; @@ -672,13 +717,14 @@ abstract class LineLayout { private VisualFragment myFragment = new VisualFragment(); private VisualOrderIterator(EditorView view, int line, - float startX, int startVisualColumn, int startLogicalColumn, int startOffset, BidiRun[] runsInVisualOrder) { + float startX, int startVisualColumn, int startOffset, BidiRun[] runsInVisualOrder) { myView = view; + myText = view == null ? null : view.getEditor().getDocument().getImmutableCharSequence(); myLine = line; + myLineStartOffset = view == null ? 0 : view.getEditor().getDocument().getLineStartOffset(line); myRuns = runsInVisualOrder; myFragment.startX = startX; myFragment.startVisualColumn = startVisualColumn; - myFragment.startLogicalColumn = startLogicalColumn; myFragment.startOffset = startOffset; } @@ -686,7 +732,7 @@ abstract class LineLayout { public boolean hasNext() { if (myRunIndex >= myRuns.length) return false; BidiRun run = myRuns[myRunIndex]; - Chunk[] chunks = run.getChunks(); + Chunk[] chunks = run.getChunks(myText, myLineStartOffset); if (myChunkIndex >= chunks.length) return false; Chunk chunk = chunks[run.isRtl() ? chunks.length - 1 - myChunkIndex : myChunkIndex]; if (myView != null) { @@ -703,19 +749,21 @@ abstract class LineLayout { BidiRun run = myRuns[myRunIndex]; if (myRunIndex == 0 && myChunkIndex == 0 && myFragmentIndex == 0) { - myFragment.startLogicalColumn += (run.isRtl() ? run.endOffset : run.startOffset) - myFragment.startOffset; + myFragment.startLogicalColumn = run.visualStartLogicalColumn; } else { - myFragment.startLogicalColumn = myFragment.getEndLogicalColumn(); if (myChunkIndex == 0 && myFragmentIndex == 0) { - myFragment.startLogicalColumn += (run.isRtl() ? run.endOffset : run.startOffset) - myFragment.getEndOffset(); + myFragment.startLogicalColumn = run.visualStartLogicalColumn; + } + else { + myFragment.startLogicalColumn = myFragment.getEndLogicalColumn(); } myFragment.startVisualColumn = myFragment.getEndVisualColumn(); myFragment.startX = myFragment.getEndX(); } myFragment.isRtl = run.isRtl(); - Chunk[] chunks = run.getChunks(); + Chunk[] chunks = run.getChunks(myText, myLineStartOffset); Chunk chunk = chunks[run.isRtl() ? chunks.length - 1 - myChunkIndex : myChunkIndex]; myFragment.delegate = chunk.fragments.get(run.isRtl() ? chunk.fragments.size() - 1 - myFragmentIndex : myFragmentIndex); myFragment.startOffset = run.isRtl() ? run.endOffset - myOffsetInsideRun : run.startOffset + myOffsetInsideRun; @@ -798,7 +846,8 @@ abstract class LineLayout { } int getLogicalColumnCount() { - return isRtl ? getLength() : delegate.getLogicalColumnCount(getMinLogicalColumn()); + // there's no need to calculate start column for RTL case - it makes sense only for TabFragment, which cannot be part of RTL run + return delegate.getLogicalColumnCount(isRtl ? 0 : getMinLogicalColumn()); } int getVisualColumnCount() { diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LogicalPositionCache.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LogicalPositionCache.java index 402a046bc3d2..a920788bbee4 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LogicalPositionCache.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LogicalPositionCache.java @@ -22,7 +22,7 @@ import com.intellij.openapi.editor.LogicalPosition; import com.intellij.openapi.editor.event.DocumentEvent; import com.intellij.openapi.editor.ex.PrioritizedDocumentListener; import com.intellij.openapi.editor.impl.EditorDocumentPriorities; -import com.intellij.util.text.CharArrayUtil; +import com.intellij.util.DocumentUtil; import org.jetbrains.annotations.NotNull; import java.util.ArrayList; @@ -66,10 +66,19 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D public void documentChanged(DocumentEvent event) { int startLine = myDocument.getLineNumber(event.getOffset()); int newEndLine = getAdjustedLineNumber(event.getOffset() + event.getNewLength()); - invalidateLines(startLine, myDocumentChangeOldEndLine, newEndLine, CharArrayUtil.indexOf(event.getNewFragment(), "\t", 0) == -1); + invalidateLines(startLine, myDocumentChangeOldEndLine, newEndLine, isSimpleText(event.getNewFragment())); myUpdateInProgress = false; } + // text for which offset<->logicalColumn conversion is trivial + private static boolean isSimpleText(@NotNull CharSequence text) { + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '\t' || c >= Character.MIN_SURROGATE && c <= Character.MAX_SURROGATE) return false; + } + return true; + } + synchronized void reset(boolean force) { checkDisposed(); int oldTabSize = myTabSize; @@ -121,6 +130,9 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D if (text.charAt(i) == '\t') { currentColumn = (currentColumn / tabSize + 1) * tabSize; } + else if (DocumentUtil.isSurrogatePair(document, i)) { + if (currentColumn == column) return i; + } else { currentColumn++; } @@ -129,6 +141,20 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D return endOffset; } + static int calcColumn(@NotNull CharSequence text, int startOffset, int startColumn, int offset, int tabSize) { + int column = startColumn; + for (int i = startOffset; i < offset; i++) { + char c = text.charAt(i); + if (c == '\t') { + column = (column / tabSize + 1) * tabSize; + } + else if (i + 1 >= text.length() || !Character.isHighSurrogate(c) || !Character.isLowSurrogate(text.charAt(i + 1))) { + column++; + } + } + return column; + } + private int getAdjustedLineNumber(int offset) { return myDocument.getTextLength() == 0 ? -1 : myDocument.getLineNumber(offset); } @@ -176,7 +202,7 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D private void checkDisposed() { if (myLines == null) myView.getEditor().throwDisposalError("Editor is already disposed"); } - + synchronized void validateState() { int lineCount = myDocument.getLineCount(); int cacheSize = myLines.size(); @@ -203,7 +229,7 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D } } - private static class LineData { + private static class LineData { private static final LineData TRIVIAL = new LineData(null); private static final int CACHE_FREQUENCY = 1024; // logical column will be cached for each CACHE_FREQUENCY-th character on the line @@ -212,7 +238,7 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D private LineData(int[] columnData) { columnCache = columnData; } - + private static LineData create(@NotNull Document document, int line, int tabSize) { int start = document.getLineStartOffset(line); int end = document.getLineEndOffset(line); @@ -220,21 +246,29 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D int[] cache = new int[cacheSize]; CharSequence text = document.getImmutableCharSequence(); int column = 0; - boolean hasTabs = false; + boolean hasTabsOrSurrogates = false; for (int i = start; i < end; i++) { if (i > start && (i - start) % CACHE_FREQUENCY == 0) { cache[(i - start) / CACHE_FREQUENCY - 1] = column; } - if (text.charAt(i) == '\t') { + char c = text.charAt(i); + if (c == '\t') { column = (column / tabSize + 1) * tabSize; - hasTabs = true; + hasTabsOrSurrogates = true; } else { + if (Character.isHighSurrogate(c)) { + hasTabsOrSurrogates = true; + if (i + 1 < text.length() && Character.isLowSurrogate(text.charAt(i + 1))) continue; + } + else { + hasTabsOrSurrogates |= Character.isLowSurrogate(c); + } column++; } } if (cacheSize > 0 && (end - start) % CACHE_FREQUENCY == 0) cache[cacheSize - 1] = column; - return hasTabs ? new LineData(cache) : TRIVIAL; + return hasTabsOrSurrogates ? new LineData(cache) : TRIVIAL; } private int offsetToLogicalColumn(@NotNull Document document, int line, int tabSize, int offset) { @@ -244,17 +278,8 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D if (columnCache == null) return relOffset; int cacheIndex = relOffset / CACHE_FREQUENCY; int startOffset = lineStartOffset + cacheIndex * CACHE_FREQUENCY; - int column = cacheIndex == 0 ? 0 : columnCache[cacheIndex - 1]; - CharSequence text = document.getImmutableCharSequence(); - for (int i = startOffset; i < offset; i++) { - if (text.charAt(i) == '\t') { - column = (column / tabSize + 1) * tabSize; - } - else { - column++; - } - } - return column; + int startColumn = cacheIndex == 0 ? 0 : columnCache[cacheIndex - 1]; + return calcColumn(document.getImmutableCharSequence(), startOffset, startColumn, offset, tabSize); } private int logicalColumnToOffset(@NotNull Document document, int line, int tabSize, int logicalColumn) { @@ -266,7 +291,10 @@ class LogicalPositionCache implements PrioritizedDocumentListener, Disposable, D result > lineEndOffset ? lineEndOffset : result; } int pos = Arrays.binarySearch(columnCache, logicalColumn); - if (pos >= 0) return lineStartOffset + (pos + 1) * CACHE_FREQUENCY; + if (pos >= 0) { + int result = lineStartOffset + (pos + 1) * CACHE_FREQUENCY; + return DocumentUtil.isInsideSurrogatePair(document, result) ? result - 1 : result; + } int startOffset = lineStartOffset + (- pos - 1) * CACHE_FREQUENCY; int column = pos == -1 ? 0 : columnCache[- pos - 2]; return calcOffset(document, logicalColumn, column, startOffset, lineEndOffset, tabSize); diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/SimpleTextFragment.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/SimpleTextFragment.java index 25d45147609e..4454c1167d71 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/SimpleTextFragment.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/SimpleTextFragment.java @@ -46,9 +46,43 @@ class SimpleTextFragment extends TextFragment { return false; } + @Override + int offsetToLogicalColumn(int offset) { + return offset; + } + @Override public void draw(Graphics2D g, float x, float y, int startColumn, int endColumn) { g.setFont(myFont); g.drawChars(myText, startColumn, endColumn - startColumn, (int)x, (int)y); - } + } + + @Override + public int getLogicalColumnCount(int startColumn) { + return myCharPositions.length; + } + + @Override + public int getVisualColumnCount(float startX) { + return myCharPositions.length; + } + + @Override + public int[] xToVisualColumn(float startX, float x) { + float relX = x - startX; + float prevPos = 0; + for (int i = 0; i < myCharPositions.length; i++) { + float newPos = myCharPositions[i]; + if (relX < (newPos + prevPos) / 2) { + return new int[] {i, relX <= prevPos ? 0 : 1}; + } + prevPos = newPos; + } + return new int[] {myCharPositions.length, relX <= myCharPositions[myCharPositions.length - 1] ? 0 : 1}; + } + + @Override + public float visualColumnToX(float startX, int column) { + return startX + getX(column); + } } diff --git a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/TextFragment.java b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/TextFragment.java index 796e5643b442..2be8da51fb3d 100644 --- a/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/TextFragment.java +++ b/platform/platform-impl/src/com/intellij/openapi/editor/impl/view/TextFragment.java @@ -36,16 +36,6 @@ abstract class TextFragment implements LineFragment { return myCharPositions.length; } - @Override - public int getLogicalColumnCount(int startColumn) { - return myCharPositions.length; - } - - @Override - public int getVisualColumnCount(float startX) { - return myCharPositions.length; - } - abstract boolean isRtl(); @NotNull @@ -58,6 +48,8 @@ abstract class TextFragment implements LineFragment { return new TextFragmentWindow(startOffset, endOffset); } + abstract int offsetToLogicalColumn(int offset); + @Override public float offsetToX(float startX, int startOffset, int offset) { return startX + getX(offset) - getX(startOffset); @@ -77,32 +69,17 @@ abstract class TextFragment implements LineFragment { return column; } - @Override - public int[] xToVisualColumn(float startX, float x) { - float relX = x - startX; - float prevPos = 0; - for (int i = 0; i < myCharPositions.length; i++) { - float newPos = myCharPositions[i]; - if (relX < (newPos + prevPos) / 2) { - return new int[] {i, relX <= prevPos ? 0 : 1}; - } - prevPos = newPos; - } - return new int[] {myCharPositions.length, relX <= myCharPositions[myCharPositions.length - 1] ? 0 : 1}; - } - - @Override - public float visualColumnToX(float startX, int column) { - return startX + getX(column); - } - private class TextFragmentWindow implements LineFragment { private final int myStartOffset; private final int myEndOffset; + private final int myStartColumn; // logical + private final int myEndColumn; // logical private TextFragmentWindow(int startOffset, int endOffset) { myStartOffset = startOffset; myEndOffset = endOffset; + myStartColumn = TextFragment.this.offsetToLogicalColumn(startOffset); + myEndColumn = TextFragment.this.offsetToLogicalColumn(endOffset); } @Override @@ -112,12 +89,12 @@ abstract class TextFragment implements LineFragment { @Override public int getLogicalColumnCount(int startColumn) { - return getLength(); + return myEndColumn - myStartColumn; } @Override public int getVisualColumnCount(float startX) { - return getLength(); + return myEndColumn - myStartColumn; } @Override @@ -132,37 +109,44 @@ abstract class TextFragment implements LineFragment { @Override public float offsetToX(float startX, int startOffset, int offset) { - return TextFragment.this.offsetToX(startX, visualColumnToParent(startOffset), visualColumnToParent(offset)); + return TextFragment.this.offsetToX(startX, visualOffsetToParent(startOffset), visualOffsetToParent(offset)); } @Override public float visualColumnToX(float startX, int column) { - return startX + getX(visualColumnToParent(column)) - getX(visualColumnToParent(0)); + int startColumnInParent = visualColumnToParent(0); + float parentStartX = startX - TextFragment.this.visualColumnToX(0, startColumnInParent); + int columnInParent = visualColumnToParent(column); + return TextFragment.this.visualColumnToX(parentStartX, columnInParent); } @Override public int[] xToVisualColumn(float startX, float x) { int startColumnInParent = visualColumnToParent(0); - float parentStartX = startX - getX(startColumnInParent); + float parentStartX = startX - TextFragment.this.visualColumnToX(0, startColumnInParent); int[] parentColumn = TextFragment.this.xToVisualColumn(parentStartX, x); int column = parentColumn[0] - startColumnInParent; - int length = getLength(); - return column < 0 ? new int[] {0, 0} : column > length ? new int[] {length, 1} : new int[] {column, parentColumn[1]}; + int columnCount = getVisualColumnCount(startX); + return column < 0 ? new int[] {0, 0} : column > columnCount ? new int[] {columnCount, 1} : new int[] {column, parentColumn[1]}; + } + + private int visualOffsetToParent(int offset) { + return offset + (isRtl() ? myCharPositions.length - myEndOffset : myStartOffset); } private int visualColumnToParent(int column) { - return column + (isRtl() ? myCharPositions.length - myEndOffset : myStartOffset); + return column + (isRtl() ? TextFragment.this.getVisualColumnCount(0) - myEndColumn : myStartColumn); } @Override - public void draw(Graphics2D g, float x, float y, int startColumn, int endColumn) { - TextFragment.this.draw(g, x, y, visualColumnToParent(startColumn), visualColumnToParent(endColumn)); + public void draw(Graphics2D g, float x, float y, int startOffset, int endOffset) { + TextFragment.this.draw(g, x, y, visualOffsetToParent(startOffset), visualOffsetToParent(endOffset)); } @NotNull @Override public LineFragment subFragment(int startOffset, int endOffset) { - return new TextFragmentWindow(startOffset + myStartOffset, endOffset + myStartOffset); + return TextFragment.this.subFragment(startOffset + myStartOffset, endOffset + myStartOffset); } } } diff --git a/platform/platform-impl/src/com/intellij/openapi/wm/impl/status/PositionPanel.java b/platform/platform-impl/src/com/intellij/openapi/wm/impl/status/PositionPanel.java index 5abb98f4439e..9a1f17715770 100644 --- a/platform/platform-impl/src/com/intellij/openapi/wm/impl/status/PositionPanel.java +++ b/platform/platform-impl/src/com/intellij/openapi/wm/impl/status/PositionPanel.java @@ -17,7 +17,10 @@ package com.intellij.openapi.wm.impl.status; import com.intellij.ide.util.GotoLineNumberDialog; import com.intellij.openapi.command.CommandProcessor; -import com.intellij.openapi.editor.*; +import com.intellij.openapi.editor.Editor; +import com.intellij.openapi.editor.EditorFactory; +import com.intellij.openapi.editor.LogicalPosition; +import com.intellij.openapi.editor.SelectionModel; import com.intellij.openapi.editor.event.*; import com.intellij.openapi.fileEditor.FileEditorManagerEvent; import com.intellij.openapi.fileEditor.ex.IdeDocumentHistory; @@ -25,17 +28,26 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.wm.StatusBar; import com.intellij.openapi.wm.StatusBarWidget; import com.intellij.ui.UIBundle; +import com.intellij.util.Alarm; import com.intellij.util.Consumer; import org.jetbrains.annotations.NotNull; +import javax.swing.*; import java.awt.*; import java.awt.event.MouseEvent; public class PositionPanel extends EditorBasedWidget implements StatusBarWidget.Multiframe, StatusBarWidget.TextPresentation, CaretListener, SelectionListener { + private static final int CHAR_COUNT_SYNC_LIMIT = 500_000; + private static final String CHAR_COUNT_UNKNOWN = "..."; + + private final Alarm myAlarm; + private CodePointCountTask myCountTask; + private String myText; public PositionPanel(@NotNull final Project project) { super(project); + myAlarm = new Alarm(Alarm.ThreadToUse.POOLED_THREAD, project); } @Override @@ -136,7 +148,17 @@ public class PositionPanel extends EditorBasedWidget implements StatusBarWidget. } } + private void updateTextWithCodePointCount(int codePointCount) { + if (myText != null) { + myText = myText.replace(CHAR_COUNT_UNKNOWN, Integer.toString(codePointCount)); + if (myStatusBar != null) { + myStatusBar.updateWidget(ID()); + } + } + } + private String getPositionText(@NotNull Editor editor) { + myCountTask = null; if (!editor.isDisposed() && myStatusBar != null) { StringBuilder message = new StringBuilder(); @@ -150,7 +172,18 @@ public class PositionPanel extends EditorBasedWidget implements StatusBarWidget. int selectionStart = selectionModel.getSelectionStart(); int selectionEnd = selectionModel.getSelectionEnd(); if (selectionEnd > selectionStart) { - message.append(UIBundle.message("position.panel.selected.chars.count", selectionEnd - selectionStart)); + CodePointCountTask countTask = new CodePointCountTask(editor.getDocument().getImmutableCharSequence(), + selectionStart, selectionEnd); + if (countTask.isQuick()) { + int charCount = countTask.calculate(); + message.append(charCount).append(' ').append(UIBundle.message("position.panel.selected.chars.count", charCount)); + } + else { + message.append(CHAR_COUNT_UNKNOWN).append(' ').append(UIBundle.message("position.panel.selected.chars.count", 2)); + myCountTask = countTask; + myAlarm.cancelAllRequests(); + myAlarm.addRequest(countTask, 0); + } int selectionStartLine = editor.getDocument().getLineNumber(selectionStart); int selectionEndLine = editor.getDocument().getLineNumber(selectionEnd); if (selectionEndLine > selectionStartLine) { @@ -170,4 +203,36 @@ public class PositionPanel extends EditorBasedWidget implements StatusBarWidget. return ""; } } + + private class CodePointCountTask implements Runnable { + private final CharSequence text; + private final int startOffset; + private final int endOffset; + + private CodePointCountTask(CharSequence text, int startOffset, int endOffset) { + this.text = text; + this.startOffset = startOffset; + this.endOffset = endOffset; + } + + private boolean isQuick() { + return endOffset - startOffset < CHAR_COUNT_SYNC_LIMIT; + } + + private int calculate() { + return Character.codePointCount(text, startOffset, endOffset); + } + + @Override + public void run() { + int count = calculate(); + //noinspection SSBasedInspection + SwingUtilities.invokeLater(() -> { + if (this == myCountTask) { + updateTextWithCodePointCount(count); + myCountTask = null; + } + }); + } + } } diff --git a/platform/platform-resources-en/src/messages/UIBundle.properties b/platform/platform-resources-en/src/messages/UIBundle.properties index d67458c8f01a..56db529a6d68 100644 --- a/platform/platform-resources-en/src/messages/UIBundle.properties +++ b/platform/platform-resources-en/src/messages/UIBundle.properties @@ -73,7 +73,7 @@ memory.usage.panel.message.text={0,number,####} of {1,number,####}M go.to.line.command.name=Go to Line go.to.line.command.double.click=Click to go to line position.panel.caret.count={0} carets -position.panel.selected.chars.count={0} {0,choice,1#char|2#chars} +position.panel.selected.chars.count={0,choice,1#char|2#chars} position.panel.selected.line.breaks.count={0} line {0,choice,1#break|2#breaks} popup.hints.panel.click.to.configure.highlighting.tooltip.text=Click to configure highlighting for this file popup.hints.panel.click.to.configure.profile.text=Click to configure inspection profiles diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/editor/actions/EditorActionTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/editor/actions/EditorActionTest.java index c1009b03199f..51e8e6b11601 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/editor/actions/EditorActionTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/editor/actions/EditorActionTest.java @@ -241,4 +241,24 @@ public class EditorActionTest extends AbstractEditorTest { executeAction(IdeActions.ACTION_EDITOR_JOIN_LINES); checkResultByText("a b c"); } + + public void testDeleteAtSurrogatePair() throws Exception { + initText("a" + SURROGATE_PAIR + "b"); + delete(); + checkResultByText("ab"); + } + + public void testBackspaceAtSurrogatePair() throws Exception { + initText("a" + SURROGATE_PAIR + "b"); + backspace(); + checkResultByText("ab"); + } + + public void testCaretMovementNearSurrogatePair() throws Exception { + initText("a" + SURROGATE_PAIR + "b"); + right(); + checkResultByText("a" + SURROGATE_PAIR + "b"); + left(); + checkResultByText("a" + SURROGATE_PAIR + "b"); + } } \ No newline at end of file diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/AbstractEditorTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/AbstractEditorTest.java index ae4bc9051171..d6886895052c 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/AbstractEditorTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/AbstractEditorTest.java @@ -52,6 +52,10 @@ public abstract class AbstractEditorTest extends LightPlatformCodeInsightTestCas public static final String LOREM_IPSUM = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."; + public static final String SURROGATE_PAIR = new String(new int[]{Character.MIN_SUPPLEMENTARY_CODE_POINT}, 0, 1); + public static final char HIGH_SURROGATE = SURROGATE_PAIR.charAt(0); + public static final char LOW_SURROGATE = SURROGATE_PAIR.charAt(1); + @Override protected void setUp() throws Exception { super.setUp(); diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorImplTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorImplTest.java index 2c5dfa36e27b..147532ee9346 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorImplTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorImplTest.java @@ -434,4 +434,22 @@ public class EditorImplTest extends AbstractEditorTest { checkResultByText("a bc"); assertTrue(myEditor.getInlayModel().hasInlineElementAt(1)); } + + public void testCoordinateConversionsAroundSurrogatePair() throws Exception { + initText("a" + SURROGATE_PAIR + "b"); + assertEquals(new LogicalPosition(0, 2), myEditor.offsetToLogicalPosition(3)); + assertEquals(3, myEditor.logicalPositionToOffset(new LogicalPosition(0, 2))); + assertEquals(new VisualPosition(0, 2), myEditor.logicalToVisualPosition(new LogicalPosition(0, 2))); + assertEquals(new LogicalPosition(0, 2), myEditor.visualToLogicalPosition(new VisualPosition(0, 2))); + } + + public void testCreationOfSurrogatePairByMergingDividedParts() throws Exception { + initText(""); // Cannot set up text with singular surrogate characters directly + runWriteCommand(() -> { + myEditor.getDocument().setText(HIGH_SURROGATE + " " + LOW_SURROGATE); + myEditor.getDocument().deleteString(1, 2); + }); + checkResultByText(SURROGATE_PAIR); + assertEquals(new LogicalPosition(0, 1), myEditor.offsetToLogicalPosition(2)); + } } diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorStressTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorStressTest.java index b0e37c7e16c3..12d3db604414 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorStressTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/editor/impl/EditorStressTest.java @@ -37,6 +37,8 @@ public class EditorStressTest extends AbstractEditorTest { private static final List ourActions = Arrays.asList(new AddText("a"), new AddText("\n"), new AddText("\t"), + new AddText(SURROGATE_PAIR.substring(0, 1)), + new AddText(SURROGATE_PAIR.substring(1, 2)), new RemoveCharacter(), new MoveCharacter(), new AddFoldRegion(), diff --git a/platform/testFramework/src/com/intellij/testFramework/EditorTestUtil.java b/platform/testFramework/src/com/intellij/testFramework/EditorTestUtil.java index 5f8310e103b7..1c59198cceec 100644 --- a/platform/testFramework/src/com/intellij/testFramework/EditorTestUtil.java +++ b/platform/testFramework/src/com/intellij/testFramework/EditorTestUtil.java @@ -226,7 +226,7 @@ public class EditorTestUtil { }); model.setEditorTextRepresentationHelper(new DefaultEditorTextRepresentationHelper(editor) { @Override - public int charWidth(char c, int fontType) { + public int charWidth(int c, int fontType) { return charWidthInPixels; } }); diff --git a/platform/testFramework/src/com/intellij/testFramework/MockFontLayoutService.java b/platform/testFramework/src/com/intellij/testFramework/MockFontLayoutService.java index a0d1da0953d9..8df99bf9f05c 100644 --- a/platform/testFramework/src/com/intellij/testFramework/MockFontLayoutService.java +++ b/platform/testFramework/src/com/intellij/testFramework/MockFontLayoutService.java @@ -51,6 +51,11 @@ public class MockFontLayoutService extends FontLayoutService { return myCharWidth; } + @Override + public int charWidth(@NotNull FontMetrics fontMetrics, int codePoint) { + return myCharWidth; + } + @Override public int getHeight(@NotNull FontMetrics fontMetrics) { return myLineHeight;