UTF-8 support added for PY-15855

python/helpers/pycharm/_bdd_utils.py

@@ -8,9 +8,8 @@ You may also need "get_what_to_run_by_env" that gets folder (current or passed a
 import os
 import time
 import abc
-import sys
 import tcmessages
-
+from utils import VersionAgnosticUtils
 
 __author__ = 'Ilya.Kazakevich'
 
@@ -226,48 +225,4 @@ class BddRunner(object):
         """
         Implement it! It should launch tests using your BDD. Use "self._" functions to report results.
         """
-        pass
-
-
-class VersionAgnosticUtils(object):
-    """
-    "six" emulator: this class fabrics appropriate tool to use regardless python version.
-    Use it to write code that works both on py2 and py3
-    """
-
-    @staticmethod
-    def __new__(cls, *more):
-        """
-        Fabrics Py2 or Py3 instance based on py version
-        """
-        real_class = _Py3KUtils if sys.version_info >= (3, 0) else _Py2Utils
-        return super(cls, real_class).__new__(real_class, *more)
-
-    def to_unicode(self, obj):
-        """
-
-        :param obj: string to convert to unicode
-        :return: unicode string
-        """
-
-        raise NotImplementedError()
-
-
-
-class _Py2Utils(VersionAgnosticUtils):
-    """
-    Util for Py2
-    """
-    def to_unicode(self, obj):
-        if isinstance(obj, unicode):
-            return obj
-        return unicode(obj.decode("utf-8"))
-
-
-
-class _Py3KUtils(VersionAgnosticUtils):
-    """
-    Util for Py3
-    """
-    def to_unicode(self, obj):
-        return str(obj)
+        pass

python/helpers/pycharm/behave_runner.py

@@ -21,7 +21,7 @@ import re
 import _bdd_utils
 from distutils import version
 from behave import __version__ as behave_version
-
+from utils import VersionAgnosticUtils
 _MAX_STEPS_SEARCH_FEATURES = 5000  # Do not look for features in folder that has more that this number of children
 _FEATURES_FOLDER = 'features'  # "features" folder name.
 
@@ -136,7 +136,7 @@ class _BehaveRunner(_bdd_utils.BddRunner):
         :param element feature/suite/step
         """
         element.location.file = element.location.filename  # To preserve _bdd_utils contract
-        utils = _bdd_utils.VersionAgnosticUtils()
+        utils = VersionAgnosticUtils()
         if isinstance(element, Step):
             # Process step
             step_name = u"{0} {1}".format(utils.to_unicode(element.keyword), utils.to_unicode(element.name))

python/helpers/pycharm/django_manage_commands_provider/__init__.py

@@ -0,0 +1 @@
+__author__ = 'Ilya.Kazakevich'

python/helpers/pycharm/django_manage_commands_provider/_parser/_argparse.py

@@ -4,7 +4,7 @@ Fetches arguments from argparse-based Django (1.8+)
 """
 from argparse import Action, _StoreTrueAction, _StoreFalseAction
 
-from _parser import _utils
+from django_manage_commands_provider._parser import _utils
 
 
 __author__ = 'Ilya.Kazakevich'

python/helpers/pycharm/django_manage_commands_provider/_parser/_optparse.py

@@ -3,7 +3,7 @@
 Fetches arguments from optparse-based Django (< 1.8)
 """
 __author__ = 'Ilya.Kazakevich'
-from _parser import _utils
+from django_manage_commands_provider._parser import _utils
 
 
 # noinspection PyUnusedLocal

python/helpers/pycharm/django_manage_commands_provider/_parser/parser.py

@@ -6,7 +6,8 @@ This module encapsulates Django semi-public API knowledge, and not very stable b
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management import ManagementUtility, get_commands, BaseCommand
 
-from _parser import _optparse, _argparse
+from django_manage_commands_provider._parser import _optparse, _argparse
+from utils import VersionAgnosticUtils
 
 
 __author__ = 'Ilya.Kazakevich'
@@ -34,7 +35,7 @@ def report_data(dumper):
         except AttributeError:
             pass
         dumper.start_command(command_name=command_name,
-                             command_help_text=str(command.usage("").replace("%prog", command_name)))
+                             command_help_text=VersionAgnosticUtils().to_unicode(command.usage("")).replace("%prog", command_name))
         module_to_use = _argparse if use_argparse else _optparse # Choose appropriate module: argparse, optparse
         module_to_use.process_command(dumper, command, command.create_parser("", command_name))
         dumper.close_command()

python/helpers/pycharm/django_manage_commands_provider/_xml.py

@@ -24,6 +24,7 @@ TODO: Since Django 1.8 we can fetch much more info from argparse like positional
 """
 from xml.dom import minidom
 from xml.dom.minidom import Element
+from utils import VersionAgnosticUtils
 
 __author__ = 'Ilya.Kazakevich'
 
@@ -91,7 +92,7 @@ class XmlDumper(object):
         :type command_args_text str
         """
         assert bool(self.__command_element), "Not in a a command"
-        self.__command_element.setAttribute("args", command_args_text)
+        self.__command_element.setAttribute("args", VersionAgnosticUtils().to_unicode(command_args_text))
 
     def add_command_option(self, long_opt_names, short_opt_names, help_text, argument_info):
         """
@@ -152,4 +153,5 @@ class XmlDumper(object):
         :return: current commands as XML as described in package
         :rtype str
         """
-        return self.__document.toprettyxml()
+        document = self.__document.toxml(encoding="utf-8")
+        return VersionAgnosticUtils().to_unicode(document.decode("utf-8") if isinstance(document, bytes) else document)

python/helpers/pycharm/manage_tasks_provider.py

@@ -9,8 +9,8 @@ Module can be called directly, but be sure env var DJANGO_SETTINGS_MODULE is set
 
 import django
 
-from _parser import parser
-import _xml
+from django_manage_commands_provider._parser import parser
+from django_manage_commands_provider import _xml
 
 
 __author__ = 'Ilya.Kazakevich'

python/helpers/pycharm/utils.py

@@ -0,0 +1,45 @@
+__author__ = 'Ilya.Kazakevich'
+import sys
+
+class VersionAgnosticUtils(object):
+    """
+    "six" emulator: this class fabrics appropriate tool to use regardless python version.
+    Use it to write code that works both on py2 and py3. # TODO: Use Six instead
+    """
+
+    @staticmethod
+    def __new__(cls, *more):
+        """
+        Fabrics Py2 or Py3 instance based on py version
+        """
+        real_class = _Py3KUtils if sys.version_info >= (3, 0) else _Py2Utils
+        return super(cls, real_class).__new__(real_class, *more)
+
+    def to_unicode(self, obj):
+        """
+
+        :param obj: string to convert to unicode
+        :return: unicode string
+        """
+
+        raise NotImplementedError()
+
+
+
+class _Py2Utils(VersionAgnosticUtils):
+    """
+    Util for Py2
+    """
+    def to_unicode(self, obj):
+        if isinstance(obj, unicode):
+            return obj
+        return unicode(obj.decode("utf-8"))
+
+
+
+class _Py3KUtils(VersionAgnosticUtils):
+    """
+    Util for Py3
+    """
+    def to_unicode(self, obj):
+        return str(obj)

python/src/com/jetbrains/python/console/PydevConsoleRunner.java

@@ -239,15 +239,24 @@ public class PydevConsoleRunner extends AbstractConsoleRunnerWithHistory<PythonC
   }
 
   public static Map<String, String> addDefaultEnvironments(Sdk sdk, Map<String, String> envs, @NotNull Project project) {
-    Charset defaultCharset = EncodingProjectManager.getInstance(project).getDefaultCharset();
-
-    final String encoding = defaultCharset.name();
-    setPythonIOEncoding(setPythonUnbuffered(envs), encoding);
+    setCorrectStdOutEncoding(envs, project);
 
     PythonSdkFlavor.initPythonPath(envs, true, PythonCommandLineState.getAddedPaths(sdk));
     return envs;
   }
 
+  /**
+   * Add requered ENV var to Python task to set its stdout charset to current project charset to allow it print correctly.
+   * @param envs map of envs to add variable
+   * @param project current project
+   */
+  public static void setCorrectStdOutEncoding(@NotNull final Map<String, String> envs, @NotNull final Project project) {
+    final Charset defaultCharset = EncodingProjectManager.getInstance(project).getDefaultCharset();
+
+    final String encoding = defaultCharset.name();
+    setPythonIOEncoding(setPythonUnbuffered(envs), encoding);
+  }
+
 
   @Override
   protected List<AnAction> fillToolBarActions(final DefaultActionGroup toolbarActions,