EDU-205 Write meaningful information to the output of a test run

python/edu/course-creator/resources/fileTemplates/internal/test_helper.py.ft

@@ -1,5 +1,6 @@
 import sys
 
+
 def get_file_text(path):
     """ get file text by path"""
     file_io = open(path, "r")
@@ -7,41 +8,65 @@ def get_file_text(path):
     file_io.close()
     return text
 
-def get_file_output(path):
-    # TODO: get file output by path
-    return ""
+
+def get_file_output():
+    saved_stdout = sys.stdout
+    try:
+        from StringIO import StringIO
+
+        out = StringIO()
+        sys.stdout = out
+        import_task_file()
+        output = out.getvalue().strip()
+        return output
+    finally:
+        sys.stdout = saved_stdout
+
 
 def test_file_importable():
     """ tests there is no obvious syntax errors"""
     path = sys.argv[-1]
+    if not path.endswith(".py"):
+        import os
+
+        parent = os.path.abspath(os.path.join(path, os.pardir))
+        python_files = [f for f in os.listdir(parent) if os.path.isfile(os.path.join(parent, f)) and f.endswith(".py")]
+        for python_file in python_files:
+            if python_file == "tests.py": continue
+            check_importable_path(os.path.join(parent, python_file))
+        return
+    check_importable_path(path)
+
+
+def check_importable_path(path):
     try:
         import_file(path)
-    except ImportError:
-        failed("File contains syntax errors")
-        return
-    except SyntaxError:
-        failed("File contains syntax errors")
-        return
-    except NameError:
-        failed("File contains syntax errors")
+    except:
+        failed("File contains syntax errors", test_file_importable.func_name)
         return
+    passed(test_file_importable.func_name)
 
-    passed()
 
 def import_file(path):
     """ returns imported file """
     if sys.version_info[0] == 2:
-      import imp
-      return imp.load_source("tmp", path)
+        import imp
+
+        return imp.load_source("tmp", path)
     elif sys.version_info[0] == 3:
-      import importlib.machinery
-      return importlib.machinery.SourceFileLoader("tmp", path).load_module("tmp")
+        import importlib.machinery
+
+        return importlib.machinery.SourceFileLoader("tmp", path).load_module("tmp")
+
 
 def import_task_file():
-    """ returns imported file """
+    """ returns imported file
+        imports file from which check action was run
+    """
     path = sys.argv[-1]
     return import_file(path)
 
+
 def test_is_not_empty():
     path = sys.argv[-1]
     file_text = get_file_text(path)
@@ -51,6 +76,7 @@ def test_is_not_empty():
     else:
         failed("The file is empty. Please, reload the task and try again.")
 
+
 def test_is_initial_text(error_text="You should modify the file"):
     path = sys.argv[-1]
     text = get_initial_text(path)
@@ -61,6 +87,7 @@ def test_is_initial_text(error_text="You should modify the file"):
     else:
         passed()
 
+
 def get_initial_text(path):
     course_lib = sys.argv[-2]
 
@@ -68,7 +95,7 @@ def get_initial_text(path):
     # path format is "project_root/lessonX/taskY/file.py"
     task_index = path.rfind(os.sep, 0, path.rfind(os.sep))
     index = path.rfind(os.sep, 0, task_index)
-    relative_path = path[index+1:]
+    relative_path = path[index + 1:]
     initial_file_path = os.path.join(course_lib, relative_path)
     return get_file_text(initial_file_path)
 
@@ -82,6 +109,7 @@ def test_text_equals(text, error_text):
     else:
         failed(error_text)
 
+
 def test_window_text_deleted(error_text="Don't just delete task text"):
     windows = get_task_windows()
 
@@ -92,17 +120,35 @@ def test_window_text_deleted(error_text="Don't just delete task text"):
     passed()
 
 
-def failed(message="Please, reload the task and try again."):
-    print("#educational_plugin FAILED + " + message)
+def failed(message="Please, reload the task and try again.", name=None):
+    if not name:
+        name = sys._getframe().f_back.f_code.co_name
+    print("#educational_plugin " + name + " FAILED + " + message)
 
-def passed():
-    print("#educational_plugin test OK")
 
-def get_task_windows():
+def passed(name=None):
+    if not name:
+        name = sys._getframe().f_back.f_code.co_name
+    print("#educational_plugin " + name + " test OK")
+
+
+def get_task_windows(file_name=None):
     prefix = "#educational_plugin_window = "
-    path = sys.argv[-1]
     import os
-    windows_path = os.path.splitext(path)[0] + "_windows"
+
+    parent = os.path.abspath(os.path.join(sys.argv[-1], os.pardir))
+    if not file_name:
+        path = sys.argv[-1]
+    else:
+        path = os.path.join(parent, file_name)
+    import os
+
+    file_name_without_extension = os.path.splitext(path)[0]
+    windows_path = file_name_without_extension + "_windows"
+    smart_test_files = [f for f in os.listdir(parent) if
+                        os.path.join(parent, f) == file_name_without_extension + "_answers_window_windows"]
+    if len(smart_test_files) == 1:
+        windows_path = os.path.join(parent, smart_test_files[0])
     windows = []
     f = open(windows_path, "r")
     window_text = ""
@@ -123,6 +169,7 @@ def get_task_windows():
     f.close()
     return windows
 
+
 def run_common_tests(error_text="Please, reload file and try again"):
     test_file_importable()
     test_is_not_empty()

python/edu/learn-python/src/com/jetbrains/python/edu/StudyTestRunner.java

@@ -16,8 +16,9 @@ import java.io.*;
 import java.util.Map;
 
 public class StudyTestRunner {
-  public static final String TEST_OK = "#educational_plugin test OK";
-  private static final String TEST_FAILED = "#educational_plugin FAILED + ";
+  public static final String STUDY_PREFIX="#educational_plugin";
+  public static final String TEST_OK = "test OK";
+  private static final String TEST_FAILED = "FAILED + ";
   private static final String PYTHONPATH = "PYTHONPATH";
   private static final Logger LOG = Logger.getInstance(StudyTestRunner.class);
   private final Task myTask;
@@ -32,7 +33,7 @@ public class StudyTestRunner {
     Sdk sdk = PythonSdkType.findPythonSdk(ModuleManager.getInstance(project).getModules()[0]);
     File testRunner = new File(myTaskDir.getPath(), myTask.getTestFile());
     GeneralCommandLine commandLine = new GeneralCommandLine();
-    commandLine.setWorkDirectory(myTaskDir.getPath());
+    commandLine.withWorkDirectory(myTaskDir.getPath());
     final Map<String, String> env = commandLine.getEnvironment();
     final VirtualFile courseDir = project.getBaseDir();
     if (courseDir != null) {
@@ -45,7 +46,12 @@ public class StudyTestRunner {
         commandLine.addParameter(testRunner.getPath());
         final Course course = StudyTaskManager.getInstance(project).getCourse();
         assert course != null;
-        commandLine.addParameter(new File(course.getResourcePath()).getParent());
+        File resourceFile = new File(course.getResourcePath());
+        String resourceFolder = resourceFile.getParent();
+        if (resourceFolder == null) {
+          return null;
+        }
+        commandLine.addParameter(resourceFolder);
         commandLine.addParameter(FileUtil.toSystemDependentName(executablePath));
         return commandLine.createProcess();
       }
@@ -60,8 +66,12 @@ public class StudyTestRunner {
     String line;
     try {
       while ((line = testOutputReader.readLine()) != null) {
-        if (line.contains(TEST_FAILED)) {
-          String res = line.substring(TEST_FAILED.length(), line.length());
+        if (line.contains(STUDY_PREFIX)) {
+          if (line.contains(TEST_OK)) {
+            continue;
+          }
+          int messageStart = line.indexOf(TEST_FAILED);
+          String res = line.substring(messageStart + TEST_FAILED.length());
           StudyUtils.closeSilently(testOutputReader);
           return res;
         }

python/edu/learn-python/src/com/jetbrains/python/edu/actions/StudyCheckAction.java

@@ -127,7 +127,7 @@ public class StudyCheckAction extends DumbAwareAction {
               return;
             }
             String failedMessage = testRunner.getPassedTests(testProcess);
-            if (failedMessage.equals(StudyTestRunner.TEST_OK)) {
+            if (failedMessage != null && failedMessage.equals(StudyTestRunner.TEST_OK)) {
               task.setStatus(StudyStatus.Solved, oldStatus);
               createTestResultPopUp("Congratulations!", MessageType.INFO.getPopupBackground(), project);
             }