Initial and incomplete Django+Buildout support: server (+debugger) only.

Consoles still lack buildout support.
PythonCommandLineState and patchers refactored.

python/helpers/pycharm/buildout_engulfer.py

@@ -0,0 +1,42 @@
+# Expects two env variables:
+# PYCHARM_ENGULF_SCRIPT = which script should be engulfed.
+# PYCHARM_PREPEND_SYSPATH = which entries should be added to the beginning of sys.path;
+#     items must be separated by path separator. May be unset.
+#
+# Given script is loaded and compiled, then sys.path is prepended as requested.
+# On win32, getpass is changed to insecure but working version.
+# Then the compiled script evaluated, as if it were run by python interpreter itself.
+# Works OK with debugger.
+
+import os
+import sys
+
+target = os.getenv("PYCHARM_ENGULF_SCRIPT")
+
+assert target, "PYCHARM_ENGULF_SCRIPT must be set"
+
+filepath = os.path.abspath(target)
+f = None
+try:
+  f = open(filepath, "r")
+  source = "\n".join((s.rstrip() for s in f.readlines()))
+finally:
+  if f:
+    f.close()
+
+from pycharm.fix_getpass import fixGetpass
+fixGetpass()
+
+#prependable = os.getenv("PYCHARM_PREPEND_SYSPATH")
+#if prependable:
+#  sys.path[0:0] = [x for x in prependable.split(os.path.pathsep)]
+
+# include engulfed's path, everyone expects this
+our_path = os.path.dirname(filepath)
+if our_path not in sys.path:
+  sys.path.append(our_path)
+
+compile(source, target, "exec")
+exec(source)
+
+# here we come

python/helpers/pycharm/django_manage.py

@@ -1,12 +1,8 @@
 #!/usr/bin/env python
-import sys
+from pycharm.fix_getpass import fixGetpass
 from django.core.management import execute_manager
 from pycharm import django_settings
 
 if __name__ == "__main__":
-    if sys.platform == 'win32':
-        import getpass
-        import warnings
-        getpass.getpass = getpass.fallback_getpass
-        warnings.simplefilter("ignore", category=getpass.GetPassWarning)
+    fixGetpass()
     execute_manager(django_settings)

python/helpers/pycharm/django_manage_tests.py

@@ -1,12 +1,8 @@
 #!/usr/bin/env python
-import sys
+from pycharm.fix_getpass import fixGetpass
 from django.core.management import execute_manager
 from pycharm import django_settings_tests
 
 if __name__ == "__main__":
-    if sys.platform == 'win32':
-        import getpass
-        import warnings
-        getpass.getpass = getpass.fallback_getpass
-        warnings.simplefilter("ignore", category=getpass.GetPassWarning)
+    fixGetpass()
     execute_manager(django_settings_tests)

python/helpers/pycharm/django_settings.py

@@ -1,3 +1,5 @@
 from settings import *
 
 TEST_RUNNER = 'pycharm.django_test_runner.run_tests'
+
+print "pycharm django settings imported" # XXX

python/helpers/pycharm/fix_getpass.py

@@ -0,0 +1,12 @@
+# on win32, redefines getpass from too clandestine to more explicit
+import sys
+def fixGetpass():
+    if sys.platform == 'win32':
+        import getpass
+        import warnings
+        fallback = getattr(getpass, 'fallback_getpass', None) # >= 2.6
+        if not fallback:
+            fallback = getpass.default_getpass # <= 2.5
+        getpass.getpass = getpass.fallback_getpass
+        warnings.simplefilter("ignore", category=getpass.GetPassWarning)
+

python/src/com/jetbrains/python/buildout/BuildoutConfigPanel.form

@@ -30,7 +30,7 @@
         </constraints>
         <properties>
           <editable value="false"/>
-          <text value=""/>
+          <text value="Set to &lt;buildout-dir&gt;/bin/django.py for proper Django support "/>
           <wrapStyleWord value="true"/>
         </properties>
         <clientProperties>

python/src/com/jetbrains/python/buildout/BuildoutFacet.java

@@ -6,8 +6,10 @@ import com.intellij.facet.FacetType;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.module.Module;
 import com.intellij.openapi.util.SystemInfo;
+import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.openapi.vfs.VfsUtil;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.VirtualFileManager;
 import com.jetbrains.appengine.facet.AppEngineFacetType;
 import com.jetbrains.python.facet.PythonPathContributingFacet;
 import org.jetbrains.annotations.NotNull;
@@ -59,6 +61,12 @@ public class BuildoutFacet extends Facet<BuildoutFacetConfiguration> implements
     return null;
   }
 
+  @Override
+  public void initFacet() {
+    BuildoutFacetConfiguration config = getConfiguration();
+    config.setPaths(extractFromScript(LocalFileSystem.getInstance().findFileByPath(config.getScriptName())));
+  }
+
   /**
    * Extracts paths from given script, assuming sys.path[0:0] assignment.
    * @param script

python/src/com/jetbrains/python/debugger/PyDebugRunner.java

@@ -3,10 +3,7 @@ package com.jetbrains.python.debugger;
 import com.intellij.execution.ExecutionException;
 import com.intellij.execution.ExecutionResult;
 import com.intellij.execution.Executor;
-import com.intellij.execution.configurations.GeneralCommandLine;
-import com.intellij.execution.configurations.ParametersList;
-import com.intellij.execution.configurations.RunProfile;
-import com.intellij.execution.configurations.RunProfileState;
+import com.intellij.execution.configurations.*;
 import com.intellij.execution.executors.DefaultDebugExecutor;
 import com.intellij.execution.runners.ExecutionEnvironment;
 import com.intellij.execution.runners.GenericProgramRunner;
@@ -61,7 +58,7 @@ public class PyDebugRunner extends GenericProgramRunner {
     final PythonCommandLineState pyState = (PythonCommandLineState)state;
     final CommandLinePatcher debug_server_patcher = new CommandLinePatcher() {
       public void patchCommandLine(GeneralCommandLine commandLine) {
-        final String[] args = new String[]{
+        final String[] debugger_args = new String[]{
           PythonHelpersLocator.getHelperPath("pydev/pydevd.py"),
           "--client", "127.0.0.1",
           "--port", String.valueOf(serverSocket.getLocalPort()),
@@ -69,17 +66,17 @@ public class PyDebugRunner extends GenericProgramRunner {
         };
         // script name is the last parameter; all other params are for python interpreter; insert just before name
         final ParametersList parameters_list = commandLine.getParametersList();
-        int parameter_offset = pyState.getInterpreterOptionsCount();
+
+        ParamsGroup debug_params = parameters_list.getParamsGroup(PythonCommandLineState.GROUP_DEBUGGER);
+        assert debug_params != null;
+        ParamsGroup exe_params = parameters_list.getParamsGroup(PythonCommandLineState.GROUP_EXE_OPTIONS);
+        assert exe_params != null;
+
         final PythonSdkFlavor flavor = pyState.getConfig().getSdkFlavor();
         if (flavor != null) {
-          final Collection<String> options = flavor.getExtraDebugOptions();
-          for (String option : options) {
-            parameters_list.addAt(parameter_offset++, option);
-          }
-        }
-        for (int i = 0; i < args.length; i++) {
-          parameters_list.addAt(i + parameter_offset, args[i]);
+          for (String option : flavor.getExtraDebugOptions()) exe_params.addParameter(option);
         }
+        for (String s : debugger_args) debug_params.addParameter(s);
       }
     };
     RunProfile profile = env.getRunProfile();

python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java

@@ -2,26 +2,19 @@ package com.jetbrains.python.run;
 
 import com.intellij.execution.configuration.EnvironmentVariablesComponent;
 import com.intellij.execution.configurations.*;
-import com.intellij.execution.runners.ProgramRunner;
 import com.intellij.openapi.module.Module;
 import com.intellij.openapi.module.ModuleManager;
-import com.intellij.openapi.options.SettingsEditor;
 import com.intellij.openapi.projectRoots.Sdk;
 import com.intellij.openapi.projectRoots.SdkType;
 import com.intellij.openapi.roots.ProjectRootManager;
 import com.intellij.openapi.util.InvalidDataException;
-import com.intellij.openapi.util.JDOMExternalizable;
 import com.intellij.openapi.util.JDOMExternalizerUtil;
 import com.intellij.openapi.util.WriteExternalException;
 import com.intellij.openapi.util.text.StringUtil;
 import com.jetbrains.python.PyBundle;
-import com.jetbrains.python.buildout.BuildoutFacet;
-import com.jetbrains.python.buildout.BuildoutFacetType;
-import com.jetbrains.python.psi.PyUtil;
 import com.jetbrains.python.sdk.PythonEnvUtil;
 import com.jetbrains.python.sdk.PythonSdkFlavor;
 import com.jetbrains.python.sdk.PythonSdkType;
-import com.jetbrains.python.toolbox.FP;
 import org.jdom.Element;
 import org.jetbrains.annotations.Nullable;
 
@@ -205,48 +198,83 @@ public abstract class AbstractPythonRunConfiguration extends ModuleBasedConfigur
     Sdk sdk = PythonSdkType.findPythonSdk(getModule());
     if (sdk != null && sdk_home != null) {
       SdkType sdk_type = sdk.getSdkType();
-      String path_value;
-      if (sdk_type instanceof PythonSdkType) {
-        File virtualenv_root = PythonSdkType.getVirtualEnvRoot(sdk_home);
-        if (virtualenv_root != null) {
-          // prepend virtualenv bin if it's not already on PATH
-          String virtualenv_bin = new File(virtualenv_root, "bin").getPath();
-          
-          if (isPassParentEnvs()) {
-            // append to PATH
-            path_value = System.getenv("PATH");
-            if (path_value != null) path_value = virtualenv_bin + File.pathSeparator + path_value;
-            else path_value = virtualenv_bin;
-          }
-          else path_value = virtualenv_bin;
-          Map<String, String> new_env = cloneEnv(commandLine.getEnvParams()); // we need a copy lest we change config's map.
-          String existing_path = new_env.get("PATH");
-          if (existing_path == null || existing_path.indexOf(virtualenv_bin) < 0) {
-            new_env.put("PATH", path_value);
-            commandLine.setEnvParams(new_env);
-          }
+      patchCommandLineFirst(commandLine, sdk_home, sdk_type);
+      patchCommandLineForVirtualenv(commandLine, sdk_home, sdk_type);
+      patchCommandLineForBuildout(commandLine, sdk_home, sdk_type);
+      patchCommandLineLast(commandLine, sdk_home, sdk_type);
+    }
+  }
+
+  /**
+   * Patches command line before virtualenv and buildout patchers.
+   * Default implementation does nothing.
+   * @param commandLine
+   * @param sdk_home
+   * @param sdk_type
+   */
+  protected void patchCommandLineFirst(GeneralCommandLine commandLine, String sdk_home, SdkType sdk_type) {
+    // override
+  }
+
+  /**
+   * Patches command line after virtualenv and buildout patchers.
+   * Default implementation does nothing.
+   * @param commandLine
+   * @param sdk_home
+   * @param sdk_type
+   */
+  protected void patchCommandLineLast(GeneralCommandLine commandLine, String sdk_home, SdkType sdk_type) {
+    // override
+  }
+
+  /**
+   * Gets called after {@link #patchCommandLineForVirtualenv(com.intellij.execution.configurations.GeneralCommandLine, String, com.intellij.openapi.projectRoots.SdkType)}
+   * Does nothing here, real implementations should use alter running script name or use engulfer.
+   * @param commandLine
+   * @param sdk_home
+   * @param sdk_type
+   */
+  protected void patchCommandLineForBuildout(GeneralCommandLine commandLine, String sdk_home, SdkType sdk_type) {
+  }
+
+  /**
+   * Alters PATH so that a virtualenv is activated, if present.
+   * @param commandLine
+   * @param sdk_home
+   * @param sdk_type
+   */
+  protected void patchCommandLineForVirtualenv(GeneralCommandLine commandLine, String sdk_home, SdkType sdk_type) {
+    String path_value;
+    if (sdk_type instanceof PythonSdkType) {
+      File virtualenv_root = PythonSdkType.getVirtualEnvRoot(sdk_home);
+      if (virtualenv_root != null) {
+        // prepend virtualenv bin if it's not already on PATH
+        String virtualenv_bin = new File(virtualenv_root, "bin").getPath();
+
+        // TODO: use system-dependent name of "PATH"
+        if (isPassParentEnvs()) {
+          // append to PATH
+          path_value = System.getenv("PATH");
+          path_value = appendToPathEnvVar(path_value, virtualenv_bin);
         }
-      }
-      Module module = getModule();
-      if (module != null) {
-        BuildoutFacet facet = BuildoutFacet.getInstance(module);
-        if (facet != null) {
-          List<String> paths = facet.getAdditionalPythonPath();
-          if (paths != null) {
-            Map<String, String> new_env = cloneEnv(commandLine.getEnvParams()); // we need a copy lest we change config's map.
-            final String PYTHONPATH = "PYTHONPATH";
-            String old_path = new_env.get(PYTHONPATH);
-            path_value = PyUtil.joinWith(File.pathSeparator, paths);
-            if (old_path != null) path_value = path_value + File.pathSeparator + old_path; 
-            new_env.put(PYTHONPATH, path_value);
-            commandLine.setEnvParams(new_env);
-          }
+        else path_value = virtualenv_bin;
+        Map<String, String> new_env = cloneEnv(commandLine.getEnvParams()); // we need a copy lest we change config's map.
+        String existing_path = new_env.get("PATH");
+        if (existing_path == null || existing_path.indexOf(virtualenv_bin) < 0) {
+          new_env.put("PATH", path_value);
+          commandLine.setEnvParams(new_env);
         }
       }
     }
   }
 
-  private static Map<String, String> cloneEnv(Map<String, String> cmd_env) {
+  protected static String appendToPathEnvVar(@Nullable String previous_value, String what_to_append) {
+    if (previous_value != null) previous_value = what_to_append + File.pathSeparator + previous_value;
+    else previous_value = what_to_append;
+    return previous_value;
+  }
+
+  protected static Map<String, String> cloneEnv(Map<String, String> cmd_env) {
     Map<String, String> new_env;
     if (cmd_env != null) new_env = new com.intellij.util.containers.HashMap<String, String>(cmd_env);
     else new_env = new com.intellij.util.containers.HashMap<String, String>();

python/src/com/jetbrains/python/run/PythonCommandLineState.java

@@ -6,6 +6,7 @@ import com.intellij.execution.ExecutionResult;
 import com.intellij.execution.Executor;
 import com.intellij.execution.configurations.CommandLineState;
 import com.intellij.execution.configurations.GeneralCommandLine;
+import com.intellij.execution.configurations.ParametersList;
 import com.intellij.execution.filters.Filter;
 import com.intellij.execution.filters.TextConsoleBuilder;
 import com.intellij.execution.filters.TextConsoleBuilderFactory;
@@ -27,6 +28,12 @@ import java.util.Map;
  * @author Leonid Shalupov
  */
 public abstract class PythonCommandLineState extends CommandLineState {
+
+  // command line has a number of fixed groups of parameters; patchers should only operate on them and not the raw list.
+
+  public static final String GROUP_EXE_OPTIONS = "Exe Options";
+  public static final String GROUP_DEBUGGER = "Debugger";
+  public static final String GROUP_SCRIPT = "Script";
   private final AbstractPythonRunConfiguration myConfig;
   private final List<Filter> myFilters;
 
@@ -100,6 +107,12 @@ public abstract class PythonCommandLineState extends CommandLineState {
 
     setRunnerPath(commandLine);
 
+    // define groups
+    ParametersList params = commandLine.getParametersList();
+    params.addParamsGroup(GROUP_EXE_OPTIONS);
+    params.addParamsGroup(GROUP_DEBUGGER);
+    params.addParamsGroup(GROUP_SCRIPT);
+
     buildCommandLineParameters(commandLine);
 
     initEnvironment(commandLine);

python/src/com/jetbrains/python/run/PythonScriptCommandLineState.java

@@ -2,6 +2,7 @@ package com.jetbrains.python.run;
 
 import com.intellij.execution.configurations.GeneralCommandLine;
 import com.intellij.execution.configurations.ParametersList;
+import com.intellij.execution.configurations.ParamsGroup;
 import com.intellij.execution.filters.Filter;
 import com.intellij.execution.runners.ExecutionEnvironment;
 import com.intellij.openapi.util.text.StringUtil;
@@ -21,13 +22,18 @@ public class PythonScriptCommandLineState extends PythonCommandLineState {
 
   @Override
   protected void buildCommandLineParameters(GeneralCommandLine commandLine) {
-    commandLine.getParametersList().addParametersString(myConfig.getInterpreterOptions());
+    ParametersList parametersList = commandLine.getParametersList();
+    ParamsGroup exe_options = parametersList.getParamsGroup(GROUP_EXE_OPTIONS);
+    assert exe_options != null;
+    exe_options.addParametersString(myConfig.getInterpreterOptions());
 
+    ParamsGroup script_parameters = parametersList.getParamsGroup(GROUP_EXE_OPTIONS);
+    assert script_parameters != null;
     if (!StringUtil.isEmptyOrSpaces(myConfig.getScriptName())) {
-      commandLine.addParameter(myConfig.getScriptName());
+      script_parameters.addParameter(myConfig.getScriptName());
     }
 
-    commandLine.getParametersList().addParametersString(myConfig.getScriptParameters());
+    script_parameters.addParametersString(myConfig.getScriptParameters());
 
     if (!StringUtil.isEmptyOrSpaces(myConfig.getWorkingDirectory())) {
       commandLine.setWorkDirectory(myConfig.getWorkingDirectory());

python/src/com/jetbrains/python/testing/PythonTestCommandLineStateBase.java

@@ -3,6 +3,7 @@ package com.jetbrains.python.testing;
 import com.intellij.execution.ExecutionException;
 import com.intellij.execution.Executor;
 import com.intellij.execution.configurations.GeneralCommandLine;
+import com.intellij.execution.configurations.ParamsGroup;
 import com.intellij.execution.filters.Filter;
 import com.intellij.execution.process.ProcessHandler;
 import com.intellij.execution.runners.ExecutionEnvironment;
@@ -65,7 +66,9 @@ public abstract class PythonTestCommandLineStateBase extends PythonCommandLineSt
       flavor.addToPythonPath(cmd, pythonPath);
     }
 
-    cmd.getParametersList().addParametersString(myConfiguration.getInterpreterOptions());
+    ParamsGroup exe_options = cmd.getParametersList().getParamsGroup(GROUP_EXE_OPTIONS);
+    assert exe_options != null;
+    exe_options.addParametersString(myConfiguration.getInterpreterOptions());
     addTestRunnerParameters(cmd);
 
     cmd.setPassParentEnvs(myConfiguration.isPassParentEnvs());

python/src/com/jetbrains/python/testing/PythonUnitTestCommandLineState.java

@@ -1,6 +1,7 @@
 package com.jetbrains.python.testing;
 
 import com.intellij.execution.configurations.GeneralCommandLine;
+import com.intellij.execution.configurations.ParamsGroup;
 import com.intellij.execution.runners.ExecutionEnvironment;
 import com.jetbrains.python.PythonHelpersLocator;
 
@@ -22,8 +23,10 @@ public class PythonUnitTestCommandLineState extends PythonTestCommandLineStateBa
   }
 
   protected void addTestRunnerParameters(GeneralCommandLine cmd) {
-    cmd.addParameter(new File(PythonHelpersLocator.getHelpersRoot(), UTRUNNER_PY).getAbsolutePath());
-    cmd.addParameters(getTestSpecs());
+    ParamsGroup script_params = cmd.getParametersList().getParamsGroup(GROUP_SCRIPT);
+    assert script_params != null;
+    script_params.addParameter(new File(PythonHelpersLocator.getHelpersRoot(), UTRUNNER_PY).getAbsolutePath());
+    script_params.addParameters(getTestSpecs());
   }
 
   @Override

python/src/com/jetbrains/python/testing/pytest/PyTestCommandLineState.java

@@ -1,6 +1,7 @@
 package com.jetbrains.python.testing.pytest;
 
 import com.intellij.execution.configurations.GeneralCommandLine;
+import com.intellij.execution.configurations.ParamsGroup;
 import com.intellij.execution.runners.ExecutionEnvironment;
 import com.intellij.openapi.util.text.StringUtil;
 import com.jetbrains.python.testing.PythonTestCommandLineStateBase;
@@ -22,11 +23,13 @@ public class PyTestCommandLineState extends PythonTestCommandLineStateBase {
   }
 
   protected void addTestRunnerParameters(GeneralCommandLine cmd) {
-    cmd.addParameters("-p", "pytest_teamcity");
-    cmd.addParameter(myConfiguration.getTestToRun());
+    ParamsGroup script_params = cmd.getParametersList().getParamsGroup(GROUP_SCRIPT);
+    assert script_params != null;
+    script_params.addParameters("-p", "pytest_teamcity");
+    script_params.addParameter(myConfiguration.getTestToRun());
     String keywords = myConfiguration.getKeywords();
     if (!StringUtil.isEmptyOrSpaces(keywords)) {
-      cmd.addParameters("-k", "\"" + keywords + "\"");
+      script_params.addParameters("-k", "\"" + keywords + "\"");
     }
   }
 }