add an inspection to find possible <clinit> deadlocks like in IDEA-147015

plugins/InspectionGadgets/InspectionGadgetsAnalysis/src/META-INF/InspectionGadgets.xml

@@ -2700,6 +2700,10 @@
                      key="while.loop.spins.on.field.display.name" groupBundle="messages.InspectionsBundle"
                      groupKey="group.names.threading.issues" enabledByDefault="false" level="WARNING"
                      implementationClass="com.siyeh.ig.threading.WhileLoopSpinsOnFieldInspection"/>
+    <localInspection groupPath="Java" language="JAVA" suppressId="StaticInitializerReferencesSubClass" shortName="StaticInitializerReferencesSubClass"
+                     bundle="com.siyeh.InspectionGadgetsBundle" key="static.initializer.references.subclass.display.name"
+                     groupBundle="messages.InspectionsBundle" groupKey="group.names.threading.issues" enabledByDefault="true"
+                     level="WARNING" implementationClass="com.siyeh.ig.threading.StaticInitializerReferencesSubClassInspection"/>
 
     <!--group.names.visibility.issues-->
     <localInspection groupPath="Java" language="JAVA" shortName="AmbiguousMethodCall" bundle="com.siyeh.InspectionGadgetsBundle" key="ambiguous.method.call.display.name"

plugins/InspectionGadgets/InspectionGadgetsAnalysis/src/com/siyeh/InspectionGadgetsBundle.properties

@@ -2042,6 +2042,7 @@ interface.may.be.annotated.functional.display.name=Interface may be annotated @F
 interface.may.be.annotated.functional.problem.descriptor=Interface <code>#ref</code> may be annotated with @FunctionalInterface
 only.report.public.methods.option=Only report 'public' methods
 lambda.parameter.hides.member.variable.display.name=Lambda parameter hides field
+static.initializer.references.subclass.display.name=Static initializer references subclass
 lambda.parameter.hides.member.variable.problem.descriptor=Lambda parameter <code>#ref</code> hides field in class ''{0}'' #loc
 shared.thread.local.random.display.name='ThreadLocalRandom' instance might be shared
 shared.thread.local.random.problem.descriptor='ThreadLocalRandom' instance might be shared between threads

plugins/InspectionGadgets/InspectionGadgetsAnalysis/src/com/siyeh/ig/threading/StaticInitializerReferencesSubClassInspection.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2000-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.siyeh.ig.threading;
+
+import com.intellij.codeInspection.BaseJavaBatchLocalInspectionTool;
+import com.intellij.codeInspection.ProblemsHolder;
+import com.intellij.openapi.util.Pair;
+import com.intellij.openapi.util.Ref;
+import com.intellij.psi.*;
+import com.intellij.psi.util.PsiUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * see https://bugs.openjdk.java.net/browse/JDK-8037567
+ * @author peter
+ */
+public class StaticInitializerReferencesSubClassInspection extends BaseJavaBatchLocalInspectionTool {
+  @NotNull
+  @Override
+  public PsiElementVisitor buildVisitor(@NotNull final ProblemsHolder holder, boolean isOnTheFly) {
+    return new JavaElementVisitor() {
+
+      @Override
+      public void visitField(PsiField field) {
+        checkSubClassReferences(field);
+      }
+
+      @Override
+      public void visitClassInitializer(PsiClassInitializer initializer) {
+        checkSubClassReferences(initializer);
+      }
+
+      private void checkSubClassReferences(PsiMember scope) {
+        if (!scope.hasModifierProperty(PsiModifier.STATIC)) return;
+
+        PsiClass containingClass = scope.getContainingClass();
+        Pair<PsiElement, PsiClass> pair = findSubClassReference(scope, containingClass);
+        if (pair != null) {
+          holder.registerProblem(pair.first,
+                                 "Referencing subclass " + pair.second.getName() + " from superclass " + containingClass.getName() + " initializer might lead to class loading deadlock");
+        }
+      }
+    };
+  }
+
+  @Nullable
+  private static Pair<PsiElement, PsiClass> findSubClassReference(@NotNull PsiElement scope, @Nullable final PsiClass baseClass) {
+    if (baseClass == null || baseClass.isInterface()) return null;
+
+    final Ref<Pair<PsiElement, PsiClass>> result = Ref.create();
+    scope.accept(new PsiRecursiveElementWalkingVisitor() {
+      @Override
+      public void visitElement(PsiElement element) {
+        if (element instanceof PsiClass || element instanceof PsiReferenceParameterList || element instanceof PsiTypeElement) return;
+
+        PsiClass targetClass = extractClass(element);
+        if (targetClass != null && !(targetClass instanceof PsiAnonymousClass) && targetClass.isInheritor(baseClass, true)) {
+          PsiElement problemElement = calcProblemElement(element);
+          if (problemElement != null) {
+            result.set(Pair.create(problemElement, targetClass));
+          }
+        }
+
+        super.visitElement(element);
+      }
+    });
+    return result.get();
+  }
+
+  @Nullable
+  private static PsiElement calcProblemElement(PsiElement element) {
+    if (element instanceof PsiNewExpression) return calcProblemElement(((PsiNewExpression)element).getClassOrAnonymousClassReference());
+    if (element instanceof PsiMethodCallExpression) return calcProblemElement(((PsiMethodCallExpression)element).getMethodExpression());
+    if (element instanceof PsiJavaCodeReferenceElement) return ((PsiJavaCodeReferenceElement)element).getReferenceNameElement();
+    return element;
+  }
+
+  @Nullable
+  private static PsiClass extractClass(PsiElement element) {
+    if (element instanceof PsiReferenceExpression) {
+      PsiElement target = ((PsiReferenceExpression)element).resolve();
+      if (target instanceof PsiClass) {
+        return (PsiClass)target;
+      }
+    }
+    if (element instanceof PsiExpression) {
+      return PsiUtil.resolveClassInClassTypeOnly(((PsiExpression)element).getType());
+    }
+    return null;
+  }
+}

plugins/InspectionGadgets/test/com/siyeh/igtest/threading/StaticInitializerReferencesSubClass/StaticInitializer.java

@@ -0,0 +1,31 @@
+class Super {
+  static Super C1 = new <warning descr="Referencing subclass Sub from superclass Super initializer might lead to class loading deadlock">Sub</warning>();
+  static Object C2 = <warning descr="Referencing subclass Sub from superclass Super initializer might lead to class loading deadlock">Sub</warning>.create();
+  static final Sub C3;
+  static final Sub C4 = SubFactory.<warning descr="Referencing subclass Sub from superclass Super initializer might lead to class loading deadlock">create</warning>();
+  static final String C5 = SubFactory.<warning descr="Referencing subclass Sub from superclass Super initializer might lead to class loading deadlock">create</warning>().toString();
+
+  static Object OK_INSIDE_ANONYMOUS = new Object() {{ Sub s = new Sub(); }};
+  static Object OK_UNRELATED = "abc";
+  static Super OK_SAME = new Super();
+  static Super OK_SAME_ANONYMOUS = new Super(){};
+  static Sub[] OK_ARRAY = new Sub[3];
+  static java.util.List<Sub> OK_GENERICS = new java.util.ArrayList<Sub>();
+
+  static {
+    C3 = new <warning descr="Referencing subclass Sub from superclass Super initializer might lead to class loading deadlock">Sub</warning>();
+  }
+}
+
+class Sub extends Super implements Intf {
+  static native Object create();
+}
+
+class SubFactory {
+  static native Sub create();
+
+}
+
+interface Intf {
+  Sub ok = new Sub();
+}

plugins/InspectionGadgets/testsrc/com/siyeh/ig/threading/StaticInitializerReferencesSubClassInspectionTest.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2000-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.siyeh.ig.threading;
+
+import com.intellij.openapi.application.PluginPathManager;
+import com.intellij.testFramework.fixtures.LightCodeInsightFixtureTestCase;
+
+/**
+ * @author peter
+ */
+public class StaticInitializerReferencesSubClassInspectionTest extends LightCodeInsightFixtureTestCase {
+  @Override
+  protected String getTestDataPath() {
+    return PluginPathManager.getPluginHomePath("InspectionGadgets") + "/test/com/siyeh/igtest/threading/StaticInitializerReferencesSubClass";
+  }
+
+  public void testStaticInitializer() {
+    myFixture.enableInspections(new StaticInitializerReferencesSubClassInspection());
+    myFixture.testHighlighting(true, false, false, getTestName(false) + ".java");
+  }
+
+}

resources-en/src/inspectionDescriptions/StaticInitializerReferencesSubClass.html

@@ -0,0 +1,7 @@
+<html>
+<body>
+This inspection reports classes that refer to their own subclasses in their static initializers or in static fields.
+Such references can cause JVM-level deadlocks in multithreaded environment, when one thread tries to load superclass
+and another thread tries to load subclass at the same time.
+</body>
+</html>