PY-29977 Don't access external doc URLs to validate them in getUrlFor()

This method is used both to check possible sources of external
documentation in DefaultDocumentationCollector#getDocumentation()
(executed inside a read action) and to show links
at the end of Quick Documentation in DocumentationComponent
(happens on EDT). In both cases network access may badly affect editor
responsiveness. The correct approach here is to provide the most
accurate URL right away in PythonDocumentationMap and
PythonDocumentationProvider implementations instead of falling back to
the documentation root.

As a result, PythonDocumentationLinkProvider#getExternalDocumentationURL
is no longer necessary and thus deprecated.

python/openapi/src/com/jetbrains/python/documentation/PythonDocumentationLinkProvider.java

@@ -29,5 +29,14 @@ public interface PythonDocumentationLinkProvider {
   @Nullable
   String getExternalDocumentationUrl(PsiElement element, PsiElement originalElement);
 
-  String getExternalDocumentationRoot(Sdk sdk);
+  /**
+   * This method was used to provide the fallback URL in case the one returned by {@link #getExternalDocumentationUrl(PsiElement, PsiElement)}
+   * doesn't exist. This check is not performed any longer to avoid UI sluggishness.
+   *
+   * @deprecated Do your best to provide a valid URL in {@link #getExternalDocumentationUrl(PsiElement, PsiElement)}
+   */
+  @Deprecated
+  default String getExternalDocumentationRoot(Sdk sdk) {
+    return "";
+  }
 }

python/src/com/jetbrains/python/documentation/PythonDocumentationMap.java

@@ -148,16 +148,6 @@ public class PythonDocumentationMap implements PersistentStateComponent<PythonDo
     return null;
   }
 
-  @Nullable
-  public String rootUrlFor(QualifiedName moduleQName) {
-    for (Entry entry : myState.myEntries) {
-      if (moduleQName.matchesPrefix(QualifiedName.fromDottedString(entry.myPrefix))) {
-        return rootForPattern(entry.myUrlPattern);
-      }
-    }
-    return null;
-  }
-
   private static String rootForPattern(String urlPattern) {
     int pos = urlPattern.indexOf('{');
     return pos >= 0 ? urlPattern.substring(0, pos) : urlPattern;

python/src/com/jetbrains/python/documentation/PythonDocumentationProvider.java

@@ -21,7 +21,6 @@ import com.intellij.psi.*;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.util.PsiTreeUtil;
 import com.intellij.psi.util.QualifiedName;
-import com.intellij.util.io.HttpRequests;
 import com.jetbrains.python.PyNames;
 import com.jetbrains.python.PyTokenTypes;
 import com.jetbrains.python.PythonDialectsTokenSetProvider;
@@ -46,7 +45,6 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.select.Elements;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -535,12 +533,12 @@ public class PythonDocumentationProvider extends AbstractDocumentationProvider i
 
   @Override
   public List<String> getUrlFor(PsiElement element, PsiElement originalElement) {
-    final String url = getUrlFor(element, originalElement, true);
+    final String url = getOnlyUrlFor(element, originalElement);
     return url == null ? null : Collections.singletonList(url);
   }
 
   @Nullable
-  public static String getUrlFor(PsiElement element, PsiElement originalElement, boolean checkExistence) {
+  public static String getOnlyUrlFor(PsiElement element, PsiElement originalElement) {
     PsiFileSystemItem file = element instanceof PsiFileSystemItem ? (PsiFileSystemItem)element : element.getContainingFile();
     if (file == null) return null;
     if (PyNames.INIT_DOT_PY.equals(file.getName())) {
@@ -568,41 +566,17 @@ public class PythonDocumentationProvider extends AbstractDocumentationProvider i
     }
     final String url = map.urlFor(qName, namedElement, pyVersion);
     if (url != null) {
-      if (checkExistence && !pageExists(url)) {
-        return map.rootUrlFor(qName);
-      }
       return url;
     }
     for (PythonDocumentationLinkProvider provider : Extensions.getExtensions(PythonDocumentationLinkProvider.EP_NAME)) {
       final String providerUrl = provider.getExternalDocumentationUrl(element, originalElement);
       if (providerUrl != null) {
-        if (checkExistence && !pageExists(providerUrl)) {
-          return provider.getExternalDocumentationRoot(sdk);
-        }
         return providerUrl;
       }
     }
     return null;
   }
 
-  private static boolean pageExists(@NotNull String url) {
-    if (new File(url).exists()) {
-      return true;
-    }
-    try {
-      HttpRequests.head(url).tryConnect();
-    }
-    catch (HttpRequests.HttpStatusException e) {
-      return false;
-    }
-    catch (IllegalArgumentException e) {
-      return false;
-    }
-    catch (IOException ignored) {
-    }
-    return true;
-  }
-
   @Nullable
   public static String pyVersion(@Nullable String versionString) {
     final String prefix = "Python ";
@@ -683,7 +657,7 @@ public class PythonDocumentationProvider extends AbstractDocumentationProvider i
 
   @Override
   public boolean hasDocumentationFor(PsiElement element, PsiElement originalElement) {
-    return getUrlFor(element, originalElement, false) != null;
+    return getOnlyUrlFor(element, originalElement) != null;
   }
 
   @Override