Run-time + static tests for Typeshed stubs

python/testData/typeshed/conftest.py

@@ -0,0 +1,14 @@
+import re
+import os
+import pip
+import pytest
+
+
+@pytest.fixture(scope='module')
+def requirements(request):
+    requirements_path = re.sub(r'(.*)_test\.py', r'\1_requirements.txt',
+                               request.module.__file__)
+    if os.path.exists(requirements_path):
+        pip.main(['install', '-r', requirements_path])
+    yield
+    # We could uninstall everything here after the module tests finish

python/testData/typeshed/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+usefixtures = requirements

python/testData/typeshed/stdlib/2and3/collections_test.py

@@ -0,0 +1,14 @@
+def test_namedtuple():
+    from collections import namedtuple
+
+    Point = namedtuple('Point', 'x y')
+    p = Point(1, 2)
+
+    assert p == Point(1, 2)
+    assert p == (1, 2)
+    assert p._replace(y=3.14).y == 3.14
+    assert p._asdict()['x'] == 1
+    assert (p.x, p.y) == (1, 2)
+    assert p[0] + p[1] == 3
+    assert p.index(1) == 0
+    assert Point._make([1, 3.14]).y == 3.14

python/testData/typeshed/third_party/2and3/six_requirements.txt

@@ -0,0 +1 @@
+six==1.10.0

python/testData/typeshed/third_party/2and3/six_test.py

@@ -0,0 +1,13 @@
+def test_python_checks():
+    from six import PY2, PY3
+
+    assert PY2 ^ PY3
+
+
+def test_xrange():
+    from six.moves import xrange
+
+    xs = xrange(5)
+    assert xs.__iter__
+    assert xs[0] == 0
+    assert sum(xs) == 10

python/testSrc/com/jetbrains/env/python/typeshed/PyTypeShedStubsTest.kt

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.env.python.typeshed
+
+import com.intellij.testFramework.EdtTestUtil
+import com.intellij.testFramework.LightPlatformTestCase
+import com.intellij.util.ThrowableRunnable
+import com.jetbrains.python.codeInsight.typing.PyTypeShed
+import com.jetbrains.python.inspections.PyTypeCheckerInspection
+import com.jetbrains.python.inspections.unresolvedReference.PyUnresolvedReferencesInspection
+import com.jetbrains.python.sdk.PythonSdkType
+import junit.framework.TestCase
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import java.io.File
+
+/**
+ * @author vlan
+ */
+@RunWith(Parameterized::class)
+class PyTypeShedStubsTest(path: String, sdkPath: String) : PyTypeShedTestCase(path, sdkPath) {
+  @Test
+  fun test() {
+    EdtTestUtil.runInEdtAndWait(ThrowableRunnable {
+      val typeShedPath = PyTypeShed.directoryPath ?: return@ThrowableRunnable
+      val importablePath = path.split("/").drop(2).joinToString("/")
+      fixture?.copyFileToProject("$typeShedPath/$path", importablePath)
+      fixture?.configureFromTempProjectFile(importablePath)
+      fixture?.enableInspections(PyUnresolvedReferencesInspection::class.java)
+      fixture?.enableInspections(PyTypeCheckerInspection::class.java)
+      fixture?.checkHighlighting(true, false, true)
+      val moduleSdk = PythonSdkType.findPythonSdk(fixture?.module)
+      TestCase.assertNotNull(moduleSdk)
+    })
+  }
+
+  companion object {
+    @Parameterized.Parameters(name = "{0}: {1}")
+    @JvmStatic fun params(): List<Array<Any>> {
+      LightPlatformTestCase.initApplication()
+      val typeShedPath = PyTypeShed.directoryPath ?: return emptyList()
+      val typeShedFile = File(typeShedPath)
+      return getSdkPaths()
+        .asSequence()
+        .flatMap { sdkPath ->
+          val level = getLanguageLevel(sdkPath) ?: return@flatMap emptySequence<Array<Any>>()
+          PyTypeShed.findRootsForLanguageLevel(level).asSequence()
+            .flatMap { root: String ->
+              File("$typeShedPath/$root").walk()
+                .filter { it.isFile && it.extension == "pyi" && "third_party" !in it.absolutePath }
+                .map { arrayOf<Any>(it.relativeTo(typeShedFile).toString(), sdkPath) }
+            }
+        }
+        .toList()
+    }
+  }
+}

python/testSrc/com/jetbrains/env/python/typeshed/PyTypeShedTestCase.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2016 JetBrains s.r.o.
+ * Copyright 2000-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.jetbrains.env.python
+package com.jetbrains.env.python.typeshed
 
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.project.Project
@@ -24,36 +24,26 @@ import com.intellij.openapi.roots.OrderRootType
 import com.intellij.openapi.roots.ProjectRootManager
 import com.intellij.openapi.vfs.StandardFileSystems
 import com.intellij.testFramework.EdtTestUtil
-import com.intellij.testFramework.LightPlatformTestCase
 import com.intellij.testFramework.fixtures.CodeInsightTestFixture
 import com.intellij.testFramework.fixtures.IdeaTestFixtureFactory
 import com.intellij.testFramework.fixtures.impl.LightTempDirTestFixtureImpl
 import com.intellij.util.ThrowableRunnable
 import com.jetbrains.env.PyEnvTaskRunner
 import com.jetbrains.env.PyEnvTestCase
-import com.jetbrains.python.codeInsight.typing.PyTypeShed
-import com.jetbrains.python.inspections.PyTypeCheckerInspection
-import com.jetbrains.python.inspections.unresolvedReference.PyUnresolvedReferencesInspection
 import com.jetbrains.python.psi.LanguageLevel
 import com.jetbrains.python.sdk.PySdkUtil
 import com.jetbrains.python.sdk.PythonSdkType
 import com.jetbrains.python.sdk.PythonSdkUpdater
 import com.jetbrains.python.sdk.flavors.PythonSdkFlavor
 import com.jetbrains.python.sdkTools.PyTestSdkTools
-import junit.framework.TestCase
 import org.junit.After
 import org.junit.Before
-import org.junit.Test
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import java.io.File
 
 /**
  * @author vlan
  */
-@RunWith(Parameterized::class)
-class PyTypeShedTest(private val path: String, private val sdkPath: String) : PyEnvTestCase() {
-  private var fixture: CodeInsightTestFixture? = null
+abstract class PyTypeShedTestCase(protected val path: String, protected val sdkPath: String) : PyEnvTestCase() {
+  protected var fixture: CodeInsightTestFixture? = null
 
   @Before
   fun initialize() {
@@ -108,48 +98,23 @@ class PyTypeShedTest(private val path: String, private val sdkPath: String) : Py
     fixture?.tearDown()
   }
 
-  @Test
-  fun test() {
-    EdtTestUtil.runInEdtAndWait(ThrowableRunnable {
-      val typeShedPath = PyTypeShed.directoryPath ?: return@ThrowableRunnable
-      val importablePath = path.split("/").drop(2).joinToString("/")
-      fixture?.copyFileToProject("$typeShedPath/$path", importablePath)
-      fixture?.configureFromTempProjectFile(importablePath)
-      fixture?.enableInspections(PyUnresolvedReferencesInspection::class.java)
-      fixture?.enableInspections(PyTypeCheckerInspection::class.java)
-      fixture?.checkHighlighting(true, false, true)
-      val moduleSdk = PythonSdkType.findPythonSdk(fixture?.module)
-      TestCase.assertNotNull(moduleSdk)
-    })
-  }
-
   companion object {
     private val sdkCache = mutableMapOf<String, Sdk>()
 
-    @Parameterized.Parameters(name = "{0}: {1}")
-    @JvmStatic fun params(): List<Array<Any>> {
-      LightPlatformTestCase.initApplication()
+    internal fun getSdkPaths(): List<String> {
       val tags = setOf("typeshed")
-      val typeShedPath = PyTypeShed.directoryPath ?: return emptyList()
-      val typeShedFile = File(typeShedPath)
       return getPythonRoots()
         .asSequence()
         .filter { PyEnvTaskRunner.isSuitableForTags(loadEnvTags(it), tags) }
         .map { PythonSdkType.getPythonExecutable(it) }
         .filterNotNull()
-        .flatMap { sdkPath ->
-          val flavor = PythonSdkFlavor.getFlavor(sdkPath) ?: return@flatMap emptySequence<Array<Any>>()
-          val versionString = flavor.getVersionString(sdkPath) ?: return@flatMap emptySequence<Array<Any>>()
-          val level = LanguageLevel.fromPythonVersion(versionString.removePrefix(flavor.name).trim())
-          PyTypeShed.findRootsForLanguageLevel(level).asSequence()
-            .flatMap { root: String ->
-              val results = File("$typeShedPath/$root").walk()
-                .filter { it.isFile && it.extension == "pyi" && "third_party" !in it.absolutePath }
-                .map { arrayOf<Any>(it.relativeTo(typeShedFile).toString(), sdkPath) }
-              results
-            }
-        }
         .toList()
     }
+
+    internal fun getLanguageLevel(sdkPath: String): LanguageLevel? {
+      val flavor = PythonSdkFlavor.getFlavor(sdkPath) ?: return null
+      val versionString = flavor.getVersionString(sdkPath) ?: return null
+      return LanguageLevel.fromPythonVersion(versionString.removePrefix(flavor.name).trim())
+    }
   }
 }

python/testSrc/com/jetbrains/env/python/typeshed/PyTypeShedTestDataTest.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.env.python.typeshed
+
+import com.intellij.execution.configurations.GeneralCommandLine
+import com.intellij.testFramework.EdtTestUtil
+import com.intellij.testFramework.LightPlatformTestCase
+import com.intellij.testFramework.PlatformTestUtil
+import com.intellij.util.ThrowableRunnable
+import com.jetbrains.python.codeInsight.typing.PyTypeShed
+import com.jetbrains.python.inspections.PyTypeCheckerInspection
+import com.jetbrains.python.inspections.unresolvedReference.PyUnresolvedReferencesInspection
+import com.jetbrains.python.sdk.PythonSdkType
+import junit.framework.TestCase
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import java.io.File
+import java.io.InputStreamReader
+
+/**
+ * @author vlan
+ */
+@RunWith(Parameterized::class)
+class PyTypeShedTestDataTest(path: String, sdkPath: String) : PyTypeShedTestCase(path, sdkPath) {
+  @Test
+  fun test() {
+    EdtTestUtil.runInEdtAndWait(ThrowableRunnable {
+      val fullPath = "$testDataPath/$path"
+      runProcess(sdkPath, "-m", "pytest", fullPath)
+      val importablePath = path.split("/").drop(2).joinToString("/")
+      fixture?.copyFileToProject(fullPath, importablePath)
+      fixture?.configureFromTempProjectFile(importablePath)
+      fixture?.enableInspections(PyUnresolvedReferencesInspection::class.java)
+      fixture?.enableInspections(PyTypeCheckerInspection::class.java)
+      fixture?.checkHighlighting(true, false, true)
+      val moduleSdk = PythonSdkType.findPythonSdk(fixture?.module)
+      TestCase.assertNotNull(moduleSdk)
+    })
+  }
+
+  private fun runProcess(vararg args: String) {
+    val process = GeneralCommandLine(args.asList()).createProcess()
+    process.waitFor()
+    val stderr = InputStreamReader(process.errorStream).readText()
+    val stdout = InputStreamReader(process.inputStream).readText()
+    TestCase.assertEquals(if (stderr.isEmpty()) stdout else stderr, 0, process.exitValue())
+  }
+
+  companion object {
+    @Parameterized.Parameters(name = "{0}: {1}")
+    @JvmStatic fun params(): List<Array<Any>> {
+      LightPlatformTestCase.initApplication()
+      val testDataFile = File(testDataPath)
+      return getSdkPaths()
+        .asSequence()
+        .flatMap { sdkPath ->
+          val level = getLanguageLevel(sdkPath) ?: return@flatMap emptySequence<Array<Any>>()
+          PyTypeShed.findRootsForLanguageLevel(level).asSequence()
+            .flatMap { root: String ->
+              File("$testDataPath/$root").walk()
+                .filter { it.isFile && it.path.matches(Regex(".*_test.py")) }
+                .map { arrayOf<Any>(it.relativeTo(testDataFile).toString(), sdkPath) }
+            }
+        }
+        .toList()
+    }
+
+    private val testDataPath: String
+      get() = "${PlatformTestUtil.getCommunityPath()}/python/testData/typeshed"
+  }
+}