don't use PSI text for calculating external system script crc (EA-125439, EA-134667)

it still remains to move this into background (IDEA-209776 Don't load file text during VFS event processing)

platform/external-system-impl/src/com/intellij/openapi/externalSystem/service/project/autoimport/ConfigurationFileCrcFactory.java

@@ -1,27 +1,18 @@
 // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.openapi.externalSystem.service.project.autoimport;
 
-import com.intellij.extapi.psi.PsiFileBase;
 import com.intellij.lang.Language;
-import com.intellij.lang.ParserDefinition;
 import com.intellij.lang.cacheBuilder.CacheBuilderRegistry;
 import com.intellij.lang.cacheBuilder.WordOccurrence;
 import com.intellij.lang.cacheBuilder.WordsScanner;
 import com.intellij.lang.findUsages.LanguageFindUsages;
-import com.intellij.lexer.Lexer;
-import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.editor.Document;
+import com.intellij.openapi.fileEditor.FileDocumentManager;
+import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
 import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.fileTypes.LanguageFileType;
-import com.intellij.openapi.fileTypes.SyntaxHighlighter;
-import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.psi.*;
-import com.intellij.psi.impl.cache.CacheUtil;
-import com.intellij.psi.impl.source.tree.LeafElement;
-import com.intellij.psi.tree.IElementType;
-import com.intellij.psi.tree.TokenSet;
-import com.intellij.util.text.CharArrayUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -31,71 +22,27 @@ import java.util.zip.CRC32;
  * @author Vladislav.Soroka
  */
 public class ConfigurationFileCrcFactory {
-  private final Project myProject;
   private final VirtualFile myFile;
 
+  @Deprecated // left for plugin compatibility
   public ConfigurationFileCrcFactory(@NotNull Project project, @NotNull VirtualFile file) {
-    myProject = project;
+    this(file);
+  }
+
+  public ConfigurationFileCrcFactory(VirtualFile file) {
     myFile = file;
   }
 
   public long create() {
-    PsiFile psiFile = PsiManager.getInstance(myProject).findFile(myFile);
-    if (psiFile != null) {
-      CRC32 crc32 = createCrcUsingLexer(psiFile);
-      if (crc32 == null) {
-        crc32 = createCrcUsingWordsScanner(psiFile);
-      }
-      if (crc32 == null) {
-        crc32 = createCrcUsingPsi(psiFile);
-      }
-      return crc32.getValue();
-    }
-    else {
-      return myFile.getModificationStamp();
-    }
-  }
+    if (myFile.isDirectory()) return myFile.getModificationStamp();
 
-  @Nullable
-  private CRC32 createCrcUsingLexer(@NotNull PsiFile psiFile) {
-    Lexer lexer;
-    TokenSet ignoredTokens;
-    if (psiFile instanceof PsiFileBase) {
-      ParserDefinition parserDefinition = ((PsiFileBase)psiFile).getParserDefinition();
-      lexer = parserDefinition.createLexer(myProject);
-      ignoredTokens = TokenSet.andSet(parserDefinition.getCommentTokens(), parserDefinition.getWhitespaceTokens());
-    }
-    else {
-      final SyntaxHighlighter syntaxHighlighter = SyntaxHighlighterFactory.getSyntaxHighlighter(psiFile.getFileType(), myProject, myFile);
-      if (syntaxHighlighter == null) return null;
-      lexer = syntaxHighlighter.getHighlightingLexer();
-      ignoredTokens = TokenSet.WHITE_SPACE;
-    }
-
-    CRC32 crc32 = new CRC32();
-    lexer.start(psiFile.getText());
-    IElementType tokenType;
-    while ((tokenType = lexer.getTokenType()) != null) {
-      if (!ignoredTokens.contains(tokenType) && !CacheUtil.isInComments(tokenType)) {
-        String tokenText = lexer.getTokenText();
-        if (!CharArrayUtil.containsOnlyWhiteSpaces(tokenText)) {
-          for (int i = 0, end = tokenText.length(); i < end; i++) {
-            crc32.update(tokenText.charAt(i));
-          }
-        }
-      }
-      lexer.advance();
-    }
-    return crc32;
-  }
-
-  @Nullable
-  private CRC32 createCrcUsingWordsScanner(@NotNull PsiFile psiFile) {
     WordsScanner wordsScanner = getScanner(myFile);
-    if (wordsScanner == null) return null;
+    if (wordsScanner == null) return myFile.getModificationStamp();
 
     CRC32 crc32 = new CRC32();
-    wordsScanner.processWords(psiFile.getText(), occurrence -> {
+    Document document = FileDocumentManager.getInstance().getCachedDocument(myFile);
+    CharSequence text = document != null ? document.getImmutableCharSequence() : LoadTextUtil.loadText(myFile);
+    wordsScanner.processWords(text, occurrence -> {
       if (occurrence.getKind() != WordOccurrence.Kind.COMMENTS) {
         CharSequence currentWord = occurrence.getBaseText().subSequence(occurrence.getStart(), occurrence.getEnd());
         for (int i = 0, end = currentWord.length(); i < end; i++) {
@@ -104,7 +51,7 @@ public class ConfigurationFileCrcFactory {
       }
       return true;
     });
-    return crc32;
+    return crc32.getValue();
   }
 
   @Nullable
@@ -121,23 +68,4 @@ public class ConfigurationFileCrcFactory {
     }
     return null;
   }
-
-  private static CRC32 createCrcUsingPsi(@NotNull PsiFile psiFile) {
-    CRC32 crc32 = new CRC32();
-    ApplicationManager.getApplication().runReadAction(() -> psiFile.acceptChildren(new PsiRecursiveElementVisitor() {
-      @Override
-      public void visitElement(PsiElement element) {
-        if (element instanceof LeafElement && !(element instanceof PsiWhiteSpace) && !(element instanceof PsiComment)) {
-          String text = element.getText();
-          if (!text.trim().isEmpty()) {
-            for (int i = 0, end = text.length(); i < end; i++) {
-              crc32.update(text.charAt(i));
-            }
-          }
-        }
-        super.visitElement(element);
-      }
-    }));
-    return crc32;
-  }
 }

platform/external-system-impl/src/com/intellij/openapi/externalSystem/service/project/autoimport/ExternalSystemProjectsWatcherImpl.java

@@ -44,7 +44,6 @@ import com.intellij.openapi.vfs.newvfs.events.VFileContentChangeEvent;
 import com.intellij.openapi.vfs.newvfs.events.VFileEvent;
 import com.intellij.openapi.vfs.pointers.VirtualFilePointer;
 import com.intellij.openapi.vfs.pointers.VirtualFilePointerManager;
-import com.intellij.psi.PsiDocumentManager;
 import com.intellij.util.PathUtil;
 import com.intellij.util.SmartList;
 import com.intellij.util.containers.ContainerUtil;
@@ -192,7 +191,6 @@ public class ExternalSystemProjectsWatcherImpl extends ExternalSystemTaskNotific
               () -> copy.forEach((document, pair) -> {
                 if (!pair.second.isValid()) return;
 
-                PsiDocumentManager.getInstance(myProject).commitDocument(document);
                 Long beforeImport = pair.second.getUserData(CRC_WITHOUT_SPACES_BEFORE_LAST_IMPORT);
                 Long current = pair.second.getUserData(CRC_WITHOUT_SPACES_CURRENT);
                 if (current != null && current.equals(beforeImport)) {
@@ -725,9 +723,8 @@ public class ExternalSystemProjectsWatcherImpl extends ExternalSystemTaskNotific
     return canonized == null ? null : FileUtil.toSystemIndependentName(canonized);
   }
 
-  @NotNull
-  private Long calculateCrc(@NotNull VirtualFile file) {
-    return new ConfigurationFileCrcFactory(myProject, file).create();
+  private static long calculateCrc(@NotNull VirtualFile file) {
+    return new ConfigurationFileCrcFactory(file).create();
   }
 
   @ApiStatus.Experimental