Avoid deadlock during index cleanup (RUBY-21411)

java/java-tests/testSrc/com/intellij/java/index/IndexTest.groovy

@@ -539,17 +539,37 @@ class IndexTest extends JavaCodeInsightFixtureTestCase {
   }
 
   private void runFindClassStubIndexQueryThatProducesInvalidResult(String qName) {
+    def foundFile = [null]
+
+    def key = qName.hashCode()
+    def searchScope = GlobalSearchScope.allScope(project)
+    def processor = new Processor<PsiFile>() {
+      @Override
+      boolean process(PsiFile file) {
+        foundFile[0] = file
+        return false
+      }
+    }
+    
     try {
-      def foundFile = [null]
 
       StubIndex.instance.
-        processElements(JavaStubIndexKeys.CLASS_FQN, qName.hashCode(), project, GlobalSearchScope.allScope(project), PsiFile.class, new Processor<PsiFile>() {
+        processElements(JavaStubIndexKeys.CLASS_FQN, key, project, searchScope, PsiClass.class, new Processor<PsiClass>() {
           @Override
-          boolean process(PsiFile file) {
-            foundFile[0] = file
+          boolean process(PsiClass aClass) {
+            StubIndex.instance.processElements(JavaStubIndexKeys.CLASS_FQN, key, project, searchScope, PsiFile.class, processor)
+
             return false
           }
         })
+      fail("Unexpected")
+    }
+    catch (AssertionError ignored) {
+      // stub mismatch
+    }
+    
+    try {      
+      StubIndex.instance.processElements(JavaStubIndexKeys.CLASS_FQN, key, project, searchScope, PsiFile.class, processor)
 
       fail("Unexpected")
     }

platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java

@@ -380,7 +380,8 @@ public class StubIndexImpl extends StubIndex implements PersistentStateComponent
 
     if (filesWithProblems != null) {
       ((FileBasedIndexImpl)FileBasedIndex.getInstance()).runCleanupAction(() -> {
-        stubUpdatingIndex.getWriteLock().lock();
+        boolean locked = stubUpdatingIndex.getWriteLock().tryLock();
+        if (!locked) return; // nested indices invokation, can not cleanup without deadlock
         try {
           Map<Key, StubIdList> artificialOldValues = new THashMap<>();
           artificialOldValues.put(key, new StubIdList());