PY-23504, PY-23303, PY-23733, PY-23647, PY-23613: Root closest to file should be used as working directory at least for pytest

If you have several roots (src, content, what ever) they all passed as sys.path. But pytest does not resolve its argument against sys.path: it uses working directory. This change does 2 things:

* If workdir not set --use closest root
* For newly created config set closest root as dir

python/src/com/jetbrains/python/testing/universalTests/PyTestRunnerUtils.kt

@@ -95,7 +95,7 @@ internal fun findPathWithPackagesByFsItem(elementPath: PsiFileSystemItem): Virtu
 /**
  * Finds root closest to some file
  */
-private fun findVFSItemRoot(virtualFile: VirtualFile, project: Project): VirtualFile? {
+internal fun findVFSItemRoot(virtualFile: VirtualFile, project: Project): VirtualFile? {
   val module = ModuleUtil.findModuleForFile(virtualFile, project)
   if (module == null) {
     Logger.getInstance(PyUniversalTestConfiguration::class.java).warn("No module for " + virtualFile)
@@ -103,16 +103,18 @@ private fun findVFSItemRoot(virtualFile: VirtualFile, project: Project): Virtual
   }
   return PyUtil.getSourceRoots(module)
            .map {
-             val path = VfsUtil.getRelativePath(virtualFile, it) ?: return null
-             com.intellij.openapi.util.Pair(path, it)
+             val path = VfsUtil.getRelativePath(virtualFile, it)
+             if (path != null) com.intellij.openapi.util.Pair(path, it)
+             else {
+               null
+             }
            }
            .filterNotNull()
            .sortedBy {
              it.first.length
            }
            .map(com.intellij.openapi.util.Pair<String, VirtualFile>::second)
-           .firstOrNull() ?: return null
-
+           .firstOrNull()
 }
 
 

python/src/com/jetbrains/python/testing/universalTests/PyUniversalPyTest.kt

@@ -21,6 +21,7 @@ import com.intellij.execution.runners.ExecutionEnvironment
 import com.intellij.openapi.options.SettingsEditor
 import com.intellij.openapi.project.Project
 import com.jetbrains.python.PythonHelper
+import com.jetbrains.python.psi.types.TypeEvalContext
 import com.jetbrains.python.testing.PythonTestConfigurationsModel
 import com.jetbrains.python.testing.VFSTestFrameworkListener
 
@@ -56,6 +57,19 @@ class PyUniversalPyTestConfiguration(project: Project, factory: PyUniversalPyTes
     }
 
   override fun isFrameworkInstalled() = VFSTestFrameworkListener.getInstance().isPyTestInstalled(sdk)
+
+  override fun getWorkingDirectorySafe(): String {
+    val dirProvidedByUser = super.getWorkingDirectory()
+    if (! dirProvidedByUser.isNullOrEmpty()) {
+      return dirProvidedByUser
+    }
+    // If dir is not set then find closest src because pytest resolves files against workdir and no sys.path
+    val module = module ?: return super.getWorkingDirectorySafe()
+    val context = TypeEvalContext.userInitiated(project, null)
+    val targetElement = target.asPsiElement(module, context, null) ?: return super.getWorkingDirectorySafe()
+    val root = findVFSItemRoot(targetElement.containingFile.virtualFile, project)?: return super.getWorkingDirectorySafe()
+    return root.path
+  }
 }
 
 object PyUniversalPyTestFactory : PyUniversalTestFactory<PyUniversalPyTestConfiguration>() {

python/src/com/jetbrains/python/testing/universalTests/PyUniversalTests.kt

@@ -562,7 +562,6 @@ object PyUniversalTestsConfigurationProducer : AbstractPythonTestConfigurationPr
               return null
             }
             if (fixPackages) {
-
               val pathResult = findPathWithPackagesByName(element)
               if (pathResult == null) {
                 Logger.getInstance(PyUniversalTestConfiguration::class.java).warn("Can't resolve")
@@ -570,6 +569,11 @@ object PyUniversalTestsConfigurationProducer : AbstractPythonTestConfigurationPr
               }
               configuration.workingDirectory = pathResult.first.path
               qualifiedName = pathResult.second.toString()
+            } else {
+              val virtualFile = element.containingFile.virtualFile
+              if (virtualFile != null) {
+                configuration.workingDirectory = findVFSItemRoot(virtualFile, element.project)?.path
+              }
             }
             return ConfigurationTarget(qualifiedName, TestTargetType.PYTHON)
           }

python/testData/testRunner/env/createConfigurationTest/test_with_src/foo/src/test_test.py

@@ -0,0 +1,2 @@
+def test_test():
+    pass

python/testSrc/com/jetbrains/env/python/testing/PythonPyTestingTest.java

@@ -3,13 +3,17 @@ package com.jetbrains.env.python.testing;
 import com.intellij.execution.configurations.RuntimeConfigurationWarning;
 import com.intellij.execution.testframework.AbstractTestProxy;
 import com.intellij.execution.testframework.sm.runner.ui.MockPrinter;
+import com.intellij.openapi.roots.ModifiableRootModel;
+import com.intellij.openapi.roots.ModuleRootManager;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiElement;
+import com.intellij.testFramework.fixtures.CodeInsightTestFixture;
 import com.intellij.util.PathUtil;
 import com.jetbrains.env.EnvTestTagsRequired;
 import com.jetbrains.env.PyEnvTestCase;
 import com.jetbrains.env.PyProcessWithConsoleTestTask;
 import com.jetbrains.env.ut.PyTestTestProcessRunner;
+import com.jetbrains.python.psi.PyFunction;
 import com.jetbrains.python.sdkTools.SdkCreationType;
 import com.jetbrains.python.testing.PythonTestConfigurationsModel;
 import com.jetbrains.python.testing.universalTests.PyUniversalPyTestConfiguration;
@@ -21,8 +25,10 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 
+import static com.jetbrains.env.ut.PyScriptTestProcessRunner.TEST_TARGET_PREFIX;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -56,14 +62,88 @@ public final class PythonPyTestingTest extends PyEnvTestCase {
                                         @NotNull String stderr,
                                         @NotNull String all) {
           Assert.assertEquals("Marker support broken", "Test tree:\n" +
-                                  "[root]\n" +
-                                  ".test_with_markers\n" +
-                                  "..test_fast(+)\n",
+                                                       "[root]\n" +
+                                                       ".test_with_markers\n" +
+                                                       "..test_fast(+)\n",
                               runner.getFormattedTestTree());
         }
       });
   }
 
+
+  /**
+   * New configuration should have closest src set as its working dir
+   */
+  @Test
+  public void testClosestSrcIsWorkDirOnNewConfig() throws Exception {
+    runPythonTest(
+      new CreateConfigurationTestTask<PyUniversalPyTestConfiguration>(PythonTestConfigurationsModel.PY_TEST_NAME,
+                                                                      PyUniversalPyTestConfiguration.class) {
+        @NotNull
+        @Override
+        protected List<PsiElement> getPsiElementsToRightClickOn() {
+          configureSrcFolder(myFixture);
+
+          myFixture.configureByFile("test_with_src/foo/src/test_test.py");
+          final PyFunction test = myFixture.findElementByText("test_test", PyFunction.class);
+          assert test != null;
+          return Collections.singletonList(test);
+        }
+
+        @Override
+        protected void checkConfiguration(@NotNull PyUniversalPyTestConfiguration configuration,
+                                          @NotNull PsiElement elementToRightClickOn) {
+          super.checkConfiguration(configuration, elementToRightClickOn);
+          Assert
+            .assertThat("Wrong configuration directory set on new config", configuration.getWorkingDirectory(), Matchers.endsWith("src"));
+        }
+      });
+  }
+
+  /**
+   * In case when workdir is not set we should use closest src
+   */
+  @Test
+  public void testClosestSrcIsWorkDirDynamically() throws Exception {
+    runPythonTest(
+      new PyProcessWithConsoleTestTask<PyTestTestProcessRunner>("/testRunner/env/createConfigurationTest/", SdkCreationType.EMPTY_SDK) {
+        @NotNull
+        @Override
+        protected PyTestTestProcessRunner createProcessRunner() throws Exception {
+          return new PyTestTestProcessRunner(TEST_TARGET_PREFIX + "test_test.test_test", 0) {
+            @Override
+            protected void configurationCreatedAndWillLaunch(@NotNull PyUniversalPyTestConfiguration configuration) throws IOException {
+              super.configurationCreatedAndWillLaunch(configuration);
+              // Reset dir to check it is calculated correctly
+              configuration.setWorkingDirectory(null);
+              configureSrcFolder(myFixture);
+              Assert
+                .assertThat("Wrong configuration directory calculated", configuration.getWorkingDirectorySafe(), Matchers.endsWith("src"));
+            }
+          };
+        }
+
+
+        @Override
+        protected void checkTestResults(@NotNull final PyTestTestProcessRunner runner,
+                                        @NotNull final String stdout,
+                                        @NotNull final String stderr,
+                                        @NotNull final String all) {
+          Assert.assertEquals("Failed to run test" + stderr, 1, runner.getPassedTestsCount());
+        }
+      });
+  }
+
+  //TODO: DOC
+  private static void configureSrcFolder(@NotNull final CodeInsightTestFixture fixture) {
+    final ModuleRootManager manager = ModuleRootManager.getInstance(fixture.getModule());
+    final ModifiableRootModel model = manager.getModifiableModel();
+    final VirtualFile srcToMark = fixture.getTempDirFixture().getFile("test_with_src/foo/src");
+    assert srcToMark != null;
+    model.addContentEntry(srcToMark);
+    model.commit();
+  }
+
   @Test
   public void testConfigurationProducer() throws Exception {
     runPythonTest(