From 28b40d3f0b93bbe058b023aae01a3486e2eda0f7 Mon Sep 17 00:00:00 2001 From: "Ilya.Kazakevich" Date: Wed, 13 May 2015 16:12:49 +0300 Subject: [PATCH] UTF-8 support added for PY-15855 --- python/helpers/pycharm/_bdd_utils.py | 49 +------------------ python/helpers/pycharm/behave_runner.py | 4 +- .../__init__.py | 1 + .../_parser/_argparse.py | 2 +- .../_parser/_optparse.py | 2 +- .../_parser/parser.py | 5 +- .../django_manage_commands_provider/_xml.py | 6 ++- .../provider.py => manage_tasks_provider.py} | 4 +- python/helpers/pycharm/utils.py | 45 +++++++++++++++++ .../python/console/PydevConsoleRunner.java | 17 +++++-- 10 files changed, 74 insertions(+), 61 deletions(-) create mode 100644 python/helpers/pycharm/django_manage_commands_provider/__init__.py rename python/helpers/pycharm/{django_manage_commands_provider/provider.py => manage_tasks_provider.py} (83%) create mode 100644 python/helpers/pycharm/utils.py diff --git a/python/helpers/pycharm/_bdd_utils.py b/python/helpers/pycharm/_bdd_utils.py index 4d4116197ff4..2bfa1be526ef 100644 --- a/python/helpers/pycharm/_bdd_utils.py +++ b/python/helpers/pycharm/_bdd_utils.py @@ -8,9 +8,8 @@ You may also need "get_what_to_run_by_env" that gets folder (current or passed a import os import time import abc -import sys import tcmessages - +from utils import VersionAgnosticUtils __author__ = 'Ilya.Kazakevich' @@ -226,48 +225,4 @@ class BddRunner(object): """ Implement it! It should launch tests using your BDD. Use "self._" functions to report results. """ - pass - - -class VersionAgnosticUtils(object): - """ - "six" emulator: this class fabrics appropriate tool to use regardless python version. - Use it to write code that works both on py2 and py3 - """ - - @staticmethod - def __new__(cls, *more): - """ - Fabrics Py2 or Py3 instance based on py version - """ - real_class = _Py3KUtils if sys.version_info >= (3, 0) else _Py2Utils - return super(cls, real_class).__new__(real_class, *more) - - def to_unicode(self, obj): - """ - - :param obj: string to convert to unicode - :return: unicode string - """ - - raise NotImplementedError() - - - -class _Py2Utils(VersionAgnosticUtils): - """ - Util for Py2 - """ - def to_unicode(self, obj): - if isinstance(obj, unicode): - return obj - return unicode(obj.decode("utf-8")) - - - -class _Py3KUtils(VersionAgnosticUtils): - """ - Util for Py3 - """ - def to_unicode(self, obj): - return str(obj) \ No newline at end of file + pass \ No newline at end of file diff --git a/python/helpers/pycharm/behave_runner.py b/python/helpers/pycharm/behave_runner.py index 7072e47e615b..ea656d8ab8f6 100644 --- a/python/helpers/pycharm/behave_runner.py +++ b/python/helpers/pycharm/behave_runner.py @@ -21,7 +21,7 @@ import re import _bdd_utils from distutils import version from behave import __version__ as behave_version - +from utils import VersionAgnosticUtils _MAX_STEPS_SEARCH_FEATURES = 5000 # Do not look for features in folder that has more that this number of children _FEATURES_FOLDER = 'features' # "features" folder name. @@ -136,7 +136,7 @@ class _BehaveRunner(_bdd_utils.BddRunner): :param element feature/suite/step """ element.location.file = element.location.filename # To preserve _bdd_utils contract - utils = _bdd_utils.VersionAgnosticUtils() + utils = VersionAgnosticUtils() if isinstance(element, Step): # Process step step_name = u"{0} {1}".format(utils.to_unicode(element.keyword), utils.to_unicode(element.name)) diff --git a/python/helpers/pycharm/django_manage_commands_provider/__init__.py b/python/helpers/pycharm/django_manage_commands_provider/__init__.py new file mode 100644 index 000000000000..a5f9f02f5908 --- /dev/null +++ b/python/helpers/pycharm/django_manage_commands_provider/__init__.py @@ -0,0 +1 @@ +__author__ = 'Ilya.Kazakevich' diff --git a/python/helpers/pycharm/django_manage_commands_provider/_parser/_argparse.py b/python/helpers/pycharm/django_manage_commands_provider/_parser/_argparse.py index b5bceee3260a..0d7f21103585 100644 --- a/python/helpers/pycharm/django_manage_commands_provider/_parser/_argparse.py +++ b/python/helpers/pycharm/django_manage_commands_provider/_parser/_argparse.py @@ -4,7 +4,7 @@ Fetches arguments from argparse-based Django (1.8+) """ from argparse import Action, _StoreTrueAction, _StoreFalseAction -from _parser import _utils +from django_manage_commands_provider._parser import _utils __author__ = 'Ilya.Kazakevich' diff --git a/python/helpers/pycharm/django_manage_commands_provider/_parser/_optparse.py b/python/helpers/pycharm/django_manage_commands_provider/_parser/_optparse.py index 31ffcc58f2ed..a04d391eb0df 100644 --- a/python/helpers/pycharm/django_manage_commands_provider/_parser/_optparse.py +++ b/python/helpers/pycharm/django_manage_commands_provider/_parser/_optparse.py @@ -3,7 +3,7 @@ Fetches arguments from optparse-based Django (< 1.8) """ __author__ = 'Ilya.Kazakevich' -from _parser import _utils +from django_manage_commands_provider._parser import _utils # noinspection PyUnusedLocal diff --git a/python/helpers/pycharm/django_manage_commands_provider/_parser/parser.py b/python/helpers/pycharm/django_manage_commands_provider/_parser/parser.py index fe84786a259c..4eb26f82fc78 100644 --- a/python/helpers/pycharm/django_manage_commands_provider/_parser/parser.py +++ b/python/helpers/pycharm/django_manage_commands_provider/_parser/parser.py @@ -6,7 +6,8 @@ This module encapsulates Django semi-public API knowledge, and not very stable b from django.core.exceptions import ImproperlyConfigured from django.core.management import ManagementUtility, get_commands, BaseCommand -from _parser import _optparse, _argparse +from django_manage_commands_provider._parser import _optparse, _argparse +from utils import VersionAgnosticUtils __author__ = 'Ilya.Kazakevich' @@ -34,7 +35,7 @@ def report_data(dumper): except AttributeError: pass dumper.start_command(command_name=command_name, - command_help_text=str(command.usage("").replace("%prog", command_name))) + command_help_text=VersionAgnosticUtils().to_unicode(command.usage("")).replace("%prog", command_name)) module_to_use = _argparse if use_argparse else _optparse # Choose appropriate module: argparse, optparse module_to_use.process_command(dumper, command, command.create_parser("", command_name)) dumper.close_command() diff --git a/python/helpers/pycharm/django_manage_commands_provider/_xml.py b/python/helpers/pycharm/django_manage_commands_provider/_xml.py index c2cebfb4df50..1baba3094d36 100644 --- a/python/helpers/pycharm/django_manage_commands_provider/_xml.py +++ b/python/helpers/pycharm/django_manage_commands_provider/_xml.py @@ -24,6 +24,7 @@ TODO: Since Django 1.8 we can fetch much more info from argparse like positional """ from xml.dom import minidom from xml.dom.minidom import Element +from utils import VersionAgnosticUtils __author__ = 'Ilya.Kazakevich' @@ -91,7 +92,7 @@ class XmlDumper(object): :type command_args_text str """ assert bool(self.__command_element), "Not in a a command" - self.__command_element.setAttribute("args", command_args_text) + self.__command_element.setAttribute("args", VersionAgnosticUtils().to_unicode(command_args_text)) def add_command_option(self, long_opt_names, short_opt_names, help_text, argument_info): """ @@ -152,4 +153,5 @@ class XmlDumper(object): :return: current commands as XML as described in package :rtype str """ - return self.__document.toprettyxml() + document = self.__document.toxml(encoding="utf-8") + return VersionAgnosticUtils().to_unicode(document.decode("utf-8") if isinstance(document, bytes) else document) diff --git a/python/helpers/pycharm/django_manage_commands_provider/provider.py b/python/helpers/pycharm/manage_tasks_provider.py similarity index 83% rename from python/helpers/pycharm/django_manage_commands_provider/provider.py rename to python/helpers/pycharm/manage_tasks_provider.py index 8f8d3d80125c..78b312643505 100644 --- a/python/helpers/pycharm/django_manage_commands_provider/provider.py +++ b/python/helpers/pycharm/manage_tasks_provider.py @@ -9,8 +9,8 @@ Module can be called directly, but be sure env var DJANGO_SETTINGS_MODULE is set import django -from _parser import parser -import _xml +from django_manage_commands_provider._parser import parser +from django_manage_commands_provider import _xml __author__ = 'Ilya.Kazakevich' diff --git a/python/helpers/pycharm/utils.py b/python/helpers/pycharm/utils.py new file mode 100644 index 000000000000..5bb432257a5e --- /dev/null +++ b/python/helpers/pycharm/utils.py @@ -0,0 +1,45 @@ +__author__ = 'Ilya.Kazakevich' +import sys + +class VersionAgnosticUtils(object): + """ + "six" emulator: this class fabrics appropriate tool to use regardless python version. + Use it to write code that works both on py2 and py3. # TODO: Use Six instead + """ + + @staticmethod + def __new__(cls, *more): + """ + Fabrics Py2 or Py3 instance based on py version + """ + real_class = _Py3KUtils if sys.version_info >= (3, 0) else _Py2Utils + return super(cls, real_class).__new__(real_class, *more) + + def to_unicode(self, obj): + """ + + :param obj: string to convert to unicode + :return: unicode string + """ + + raise NotImplementedError() + + + +class _Py2Utils(VersionAgnosticUtils): + """ + Util for Py2 + """ + def to_unicode(self, obj): + if isinstance(obj, unicode): + return obj + return unicode(obj.decode("utf-8")) + + + +class _Py3KUtils(VersionAgnosticUtils): + """ + Util for Py3 + """ + def to_unicode(self, obj): + return str(obj) \ No newline at end of file diff --git a/python/src/com/jetbrains/python/console/PydevConsoleRunner.java b/python/src/com/jetbrains/python/console/PydevConsoleRunner.java index 414e1d30d00d..f5f5446b8002 100644 --- a/python/src/com/jetbrains/python/console/PydevConsoleRunner.java +++ b/python/src/com/jetbrains/python/console/PydevConsoleRunner.java @@ -239,15 +239,24 @@ public class PydevConsoleRunner extends AbstractConsoleRunnerWithHistory addDefaultEnvironments(Sdk sdk, Map envs, @NotNull Project project) { - Charset defaultCharset = EncodingProjectManager.getInstance(project).getDefaultCharset(); - - final String encoding = defaultCharset.name(); - setPythonIOEncoding(setPythonUnbuffered(envs), encoding); + setCorrectStdOutEncoding(envs, project); PythonSdkFlavor.initPythonPath(envs, true, PythonCommandLineState.getAddedPaths(sdk)); return envs; } + /** + * Add requered ENV var to Python task to set its stdout charset to current project charset to allow it print correctly. + * @param envs map of envs to add variable + * @param project current project + */ + public static void setCorrectStdOutEncoding(@NotNull final Map envs, @NotNull final Project project) { + final Charset defaultCharset = EncodingProjectManager.getInstance(project).getDefaultCharset(); + + final String encoding = defaultCharset.name(); + setPythonIOEncoding(setPythonUnbuffered(envs), encoding); + } + @Override protected List fillToolBarActions(final DefaultActionGroup toolbarActions,