apply bidi layout algorithm only within lexical entities

platform/platform-impl/src/com/intellij/openapi/editor/impl/view/EditorView.java

@@ -15,26 +15,43 @@
  */
 package com.intellij.openapi.editor.impl.view;
 
+import com.intellij.lang.Language;
+import com.intellij.lang.LanguageParserDefinitions;
+import com.intellij.lang.ParserDefinition;
+import com.intellij.lexer.Lexer;
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.application.ApplicationManager;
-import com.intellij.openapi.editor.FoldRegion;
+import com.intellij.openapi.editor.*;
-import com.intellij.openapi.editor.LogicalPosition;
-import com.intellij.openapi.editor.VisualPosition;
 import com.intellij.openapi.editor.colors.EditorColorsScheme;
 import com.intellij.openapi.editor.colors.EditorFontType;
+import com.intellij.openapi.editor.colors.TextAttributesKey;
 import com.intellij.openapi.editor.ex.DocumentEx;
 import com.intellij.openapi.editor.ex.util.EditorUIUtil;
 import com.intellij.openapi.editor.ex.util.EditorUtil;
+import com.intellij.openapi.editor.ex.util.LexerEditorHighlighter;
+import com.intellij.openapi.editor.highlighter.EditorHighlighter;
+import com.intellij.openapi.editor.highlighter.HighlighterClient;
 import com.intellij.openapi.editor.impl.EditorImpl;
 import com.intellij.openapi.editor.markup.TextAttributes;
+import com.intellij.openapi.fileEditor.FileDocumentManager;
+import com.intellij.openapi.fileTypes.LanguageFileType;
+import com.intellij.openapi.fileTypes.SyntaxHighlighterBase;
+import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.Disposer;
 import com.intellij.openapi.util.Key;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.LanguageSubstitutors;
+import com.intellij.psi.tree.IElementType;
 import com.intellij.util.ui.UIUtil;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.awt.*;
 import java.awt.font.FontRenderContext;
 import java.awt.image.BufferedImage;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
 
 /**
  * A facade for components responsible for drawing editor contents, managing editor size 
@@ -53,7 +70,8 @@ public class EditorView implements Disposable {
   private final EditorSizeManager mySizeManager;
   private final TextLayoutCache myTextLayoutCache;
   private final TabFragment myTabFragment;
-    
+  
+  private EditorHighlighter myHighlighter; // accessed only in EDT
   private String myPrefixText; // accessed only in EDT
   private LineLayout myPrefixLayout; // guarded by myLock
   private TextAttributes myPrefixAttributes; // accessed only in EDT
@@ -76,12 +94,72 @@ public class EditorView implements Disposable {
     mySizeManager = new EditorSizeManager(this);
     myTextLayoutCache = new TextLayoutCache(this);
     myTabFragment = new TabFragment(this);
+    
     Disposer.register(this, myTextLayoutCache);
     Disposer.register(this, mySizeManager);
     
     reinitSettings();
   }
   
+  private void disposeHighlighter() {
+    if (myHighlighter != null) {
+      myDocument.removeDocumentListener(myHighlighter);
+      myHighlighter = null;
+    }
+  }
+  
+  private void updateHighlighter() {
+    disposeHighlighter();
+    myHighlighter = createHighlighter(myEditor);
+    if (myHighlighter != null) {
+      myDocument.addDocumentListener(myHighlighter);
+    }
+  }
+
+  @Nullable
+  private static EditorHighlighter createHighlighter(Editor editor) {
+    final Project project = editor.getProject();
+    if (project == null) return null;
+    final Document document = editor.getDocument();
+    VirtualFile file = FileDocumentManager.getInstance().getFile(document);
+    if (file == null || !(file.getFileType() instanceof LanguageFileType)) return null;
+    Language draftLanguage = ((LanguageFileType)file.getFileType()).getLanguage();
+    Language language = LanguageSubstitutors.INSTANCE.substituteLanguage(draftLanguage, file, project);
+    ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
+    if (parserDefinition == null) return null;
+    final Lexer lexer = parserDefinition.createLexer(project);
+    LexerEditorHighlighter highlighter = new LexerEditorHighlighter(new SyntaxHighlighterBase() {
+      @Override
+      public
+      @NotNull
+      Lexer getHighlightingLexer() {
+        return lexer;
+      }
+
+      @NotNull
+      @Override
+      public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
+        return EMPTY;
+      }
+    }, DUMMY_COLORS_SCHEME);
+    highlighter.setEditor(new HighlighterClient() {
+      @Override
+      public Project getProject() {
+        return project;
+      }
+
+      @Override
+      public void repaint(int start, int end) {}
+
+      @Override
+      public Document getDocument() {
+        return document;
+      }
+    });
+    highlighter.setText(document.getImmutableCharSequence());
+    return highlighter;
+  }
+
   EditorImpl getEditor() {
     return myEditor;
   }
@@ -102,8 +180,14 @@ public class EditorView implements Disposable {
     return myTabFragment;
   }
 
+  @Nullable
+  EditorHighlighter getHighlighter() {
+    return myHighlighter;
+  }
+
   @Override
   public void dispose() {
+    disposeHighlighter();
   }
 
   public int yToVisualLine(int y) {
@@ -240,6 +324,7 @@ public class EditorView implements Disposable {
     reset();
     setPrefix(myPrefixText, myPrefixAttributes); // recreate prefix layout
     invalidateFoldRegionLayouts();
+    updateHighlighter();
   }
   
   public void invalidateRange(int startOffset, int endOffset) {
@@ -397,4 +482,14 @@ public class EditorView implements Disposable {
   private static void assertIsReadAccess() {
     ApplicationManager.getApplication().assertReadAccessAllowed();
   }
+  
+  private static final EditorColorsScheme DUMMY_COLORS_SCHEME = (EditorColorsScheme)
+    Proxy.newProxyInstance(EditorView.class.getClassLoader(), 
+                           new Class[]{EditorColorsScheme.class}, 
+                           new InvocationHandler() {
+                             @Override
+                             public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+                               throw new UnsupportedOperationException();
+                             }
+                           });
 }

platform/platform-impl/src/com/intellij/openapi/editor/impl/view/LineLayout.java

@@ -16,8 +16,11 @@
 package com.intellij.openapi.editor.impl.view;
 
 import com.intellij.openapi.editor.colors.FontPreferences;
+import com.intellij.openapi.editor.highlighter.EditorHighlighter;
+import com.intellij.openapi.editor.highlighter.HighlighterIterator;
 import com.intellij.openapi.editor.impl.ComplementaryFontsRegistry;
 import com.intellij.openapi.editor.impl.EditorImpl;
+import com.intellij.psi.tree.IElementType;
 import com.intellij.util.text.CharArrayUtil;
 import org.intellij.lang.annotations.JdkConstants;
 import org.jetbrains.annotations.NotNull;
@@ -80,7 +83,7 @@ class LineLayout {
     EditorImpl editor = view.getEditor();
     FontPreferences fontPreferences = editor.getColorsScheme().getFontPreferences();
     char[] chars = CharArrayUtil.fromSequence(editor.getDocument().getImmutableCharSequence(), lineStartOffset, lineEndOffset);
-    List<BidiRun> runs = createRuns(editor, chars);
+    List<BidiRun> runs = createRuns(view, chars, lineStartOffset);
     for (BidiRun run : runs) {
       IterationState it = new IterationState(editor, lineStartOffset + run.startOffset, lineStartOffset + run.endOffset, 
                                              false, false, false, false);
@@ -100,7 +103,7 @@ class LineLayout {
     EditorImpl editor = view.getEditor();
     FontPreferences fontPreferences = editor.getColorsScheme().getFontPreferences();
     char[] chars = CharArrayUtil.fromSequence(text);
-    List<BidiRun> runs = createRuns(editor, chars);
+    List<BidiRun> runs = createRuns(view, chars, -1);
     for (BidiRun run : runs) {
       addFragments(run, chars, run.startOffset, run.endOffset, fontStyle, fontPreferences, fontRenderContext, null);
       assert !run.fragments.isEmpty();
@@ -108,17 +111,56 @@ class LineLayout {
     return runs;
   }
   
-  private static List<BidiRun> createRuns(EditorImpl editor, char[] text) {
+  private static List<BidiRun> createRuns(EditorView view, char[] text, int startOffsetInEditor) {
-    if (editor.myDisableRtl) return Collections.singletonList(new BidiRun((byte)0, 0, text.length));
+    int textLength = text.length;
-    Bidi bidi = new Bidi(text, 0, null, 0, text.length, Bidi.DIRECTION_LEFT_TO_RIGHT);
+    if (view.getEditor().myDisableRtl) return Collections.singletonList(new BidiRun((byte)0, 0, textLength));
-    int runCount = bidi.getRunCount();
+    List<BidiRun> runs = new ArrayList<BidiRun>();
-    List<BidiRun> runs = new ArrayList<BidiRun>(runCount);
+    EditorHighlighter highlighter = view.getHighlighter();
-    for (int i = 0; i < runCount; i++) {
+    if (startOffsetInEditor >= 0 && highlighter != null) {
-      runs.add(new BidiRun((byte)bidi.getRunLevel(i), bidi.getRunStart(i), bidi.getRunLimit(i)));
+      // running bidi algorithm separately for text fragments corresponding to different lexer tokens
+      int lastOffset = startOffsetInEditor;
+      IElementType lastToken = null;
+      HighlighterIterator iterator = highlighter.createIterator(startOffsetInEditor);
+      int endOffsetInEditor = startOffsetInEditor + textLength;
+      while (!iterator.atEnd() && iterator.getStart() < endOffsetInEditor) {
+        IElementType currentToken = iterator.getTokenType();
+        if (currentToken != lastToken) {
+          int tokenStart = Math.max(iterator.getStart(), startOffsetInEditor);
+          addRuns(runs, text, lastOffset - startOffsetInEditor, tokenStart - startOffsetInEditor);
+          lastToken = currentToken;
+          lastOffset = tokenStart;
+        }
+        iterator.advance();
+      }
+      addRuns(runs, text, lastOffset - startOffsetInEditor, endOffsetInEditor - startOffsetInEditor);
+    }
+    else {
+      addRuns(runs, text, 0, textLength);
     }
     return runs;
   }
   
+  private static void addRuns(List<BidiRun> runs, char[] text, int start, int end) {
+    if (start >= end) return;
+    Bidi bidi = new Bidi(text, start, null, 0, end - start, Bidi.DIRECTION_LEFT_TO_RIGHT);
+    int runCount = bidi.getRunCount();
+    for (int i = 0; i < runCount; i++) {
+      addOrMergeRun(runs, new BidiRun((byte)bidi.getRunLevel(i), start + bidi.getRunStart(i), start + bidi.getRunLimit(i)));
+    }
+  }
+
+  private static void addOrMergeRun(List<BidiRun> runs, BidiRun run) {
+    int size = runs.size();
+    if (size > 0 && runs.get(size - 1).level == 0 && run.level == 0) {
+      BidiRun lastRun = runs.remove(size - 1);
+      assert lastRun.endOffset == run.startOffset;
+      runs.add(new BidiRun((byte)0, lastRun.startOffset, run.endOffset));
+    }
+    else {
+      runs.add(run);
+    }
+  }
+  
   private static void addFragments(BidiRun run, char[] text, int start, int end, int fontStyle,
                                    FontPreferences fontPreferences, FontRenderContext fontRenderContext, 
                                    @Nullable TabFragment tabFragment) {