diff --git a/python/src/com/jetbrains/python/buildout/BuildoutConfigurable.java b/python/src/com/jetbrains/python/buildout/BuildoutConfigurable.java index e4c6626a8321..159f983fedea 100644 --- a/python/src/com/jetbrains/python/buildout/BuildoutConfigurable.java +++ b/python/src/com/jetbrains/python/buildout/BuildoutConfigurable.java @@ -2,6 +2,7 @@ package com.jetbrains.python.buildout; import com.intellij.facet.FacetManager; import com.intellij.facet.ModifiableFacetModel; +import com.intellij.ide.highlighter.ArchiveFileType; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleManager; @@ -14,6 +15,7 @@ import com.intellij.openapi.roots.impl.OrderEntryUtil; import com.intellij.openapi.roots.libraries.Library; import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.JarFileSystem; import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.ui.UIUtil; @@ -26,7 +28,10 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.FocusEvent; import java.awt.event.FocusListener; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * A silly configurable to add buildout facet configurator to PyCharm @@ -120,7 +125,7 @@ public class BuildoutConfigurable implements Configurable, NonDefaultProjectConf if (script_file == null || script_file.isDirectory()) { throw new ConfigurationException("Invalid script file '" + script_name + "'"); } - paths_from_script = BuildoutFacet.extractFromScript(script_file); + paths_from_script = BuildoutFacet.extractBuildoutPaths(script_file); if (paths_from_script == null) { throw new ConfigurationException("Failed to extract paths from '" + script_file.getPresentableName() + "'"); } @@ -147,7 +152,7 @@ public class BuildoutConfigurable implements Configurable, NonDefaultProjectConf // update existing Library lib = orderEntry.getLibrary(); if (lib != null) { - fillLibrary(lib, paths); + fillLibrary(module.getProject(), lib, paths); return; } } @@ -158,21 +163,38 @@ public class BuildoutConfigurable implements Configurable, NonDefaultProjectConf .getLibraryTable(root_model.getProject()) .createLibrary(BUILDOUT_LIB_NAME) ; - fillLibrary(lib, paths); + fillLibrary(module.getProject(), lib, paths); root_model.addLibraryEntry(lib); root_model.commit(); } }); } - private static void fillLibrary(Library lib, List paths) { + private static void fillLibrary(Project project, Library lib, List paths) { Library.ModifiableModel modifiableModel = lib.getModifiableModel(); for (String root : lib.getUrls(OrderRootType.CLASSES)) { modifiableModel.removeRoot(root, OrderRootType.CLASSES); } + Set roots = new HashSet(); + ProjectRootManager rootManager = ProjectRootManager.getInstance(project); + Collections.addAll(roots, rootManager.getContentRoots()); + Collections.addAll(roots, rootManager.getContentSourceRoots()); if (paths != null) { for (String dir : paths) { - modifiableModel.addRoot("file://"+dir, OrderRootType.CLASSES); + VirtualFile pathEntry = LocalFileSystem.getInstance().findFileByPath(dir); + if (pathEntry != null && !pathEntry.isDirectory() && pathEntry.getFileType() instanceof ArchiveFileType) { + pathEntry = JarFileSystem.getInstance().getJarRootForLocalFile(pathEntry); + } + // buildout includes source root of project in paths; don't add it as library home + if (pathEntry != null && roots.contains(pathEntry)) { + continue; + } + if (pathEntry != null) { + modifiableModel.addRoot(pathEntry, OrderRootType.CLASSES); + } + else { + modifiableModel.addRoot("file://"+dir, OrderRootType.CLASSES); + } } } modifiableModel.commit(); diff --git a/python/src/com/jetbrains/python/buildout/BuildoutFacet.java b/python/src/com/jetbrains/python/buildout/BuildoutFacet.java index 4f8947343b69..00eadc01aa24 100644 --- a/python/src/com/jetbrains/python/buildout/BuildoutFacet.java +++ b/python/src/com/jetbrains/python/buildout/BuildoutFacet.java @@ -11,6 +11,7 @@ import com.intellij.openapi.module.Module; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.LineTokenizer; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.*; import com.intellij.psi.PsiFile; @@ -42,6 +43,7 @@ public class BuildoutFacet extends Facet implements private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.buildout.BuildoutFacet"); @NonNls public static final String BUILDOUT_CFG = "buildout.cfg"; + @NonNls public static final String SCRIPT_SUFFIX = "-script"; public BuildoutFacet(@NotNull final FacetType facetType, @NotNull final Module module, @@ -113,7 +115,7 @@ public class BuildoutFacet extends Facet implements BuildoutFacetConfiguration config = getConfiguration(); final VirtualFile script = getScript(); if (script != null) { - config.setPaths(extractFromScript(script)); + config.setPaths(extractBuildoutPaths(script)); } } @@ -122,6 +124,29 @@ public class BuildoutFacet extends Facet implements return LocalFileSystem.getInstance().findFileByPath(getConfiguration().getScriptName()); } + @Nullable + public static List extractBuildoutPaths(@NotNull VirtualFile script) { + try { + List paths = extractFromScript(script); + if (paths == null) { + VirtualFile root = script.getParent().getParent(); + String partName = FileUtil.getNameWithoutExtension(script.getName()); + if (SystemInfo.isWindows && partName.endsWith(SCRIPT_SUFFIX)) { + partName = partName.substring(0, partName.length() - SCRIPT_SUFFIX.length()); + } + VirtualFile sitePy = root.findFileByRelativePath("parts/" + partName + "/site.py"); + if (sitePy != null) { + paths = extractFromSitePy(sitePy); + } + } + return paths; + } + catch(IOException e) { + LOG.info(e); + return null; + } + } + /** * Extracts paths from given script, assuming sys.path[0:0] assignment. * @@ -129,39 +154,61 @@ public class BuildoutFacet extends Facet implements * @return extracted paths, or null if extraction fails. */ @Nullable - public static List extractFromScript(VirtualFile script) { - assert script != null; - List ret = new LinkedList(); - try { - String text = VfsUtil.loadText(script); - Pattern pat = Pattern.compile("(?:^\\s*(['\"])(.*)(\\1),\\s*$)|(\\])", Pattern.MULTILINE); - final String bait_string = "sys.path[0:0]"; - int pos = text.indexOf(bait_string); - if (pos >= 0) { - pos += bait_string.length(); - Matcher scanner = pat.matcher(text); - boolean did_nothing = true; - while (scanner.find(pos)) { - did_nothing = false; - String value = scanner.group(2); - if (value != null) { - ret.add(value); - pos = scanner.end(); + public static List extractFromScript(@NotNull VirtualFile script) throws IOException { + String text = VfsUtil.loadText(script); + Pattern pat = Pattern.compile("(?:^\\s*(['\"])(.*)(\\1),\\s*$)|(\\])", Pattern.MULTILINE); + final String bait_string = "sys.path[0:0]"; + int pos = text.indexOf(bait_string); + List ret = null; + if (pos >= 0) { + pos += bait_string.length(); + Matcher scanner = pat.matcher(text); + while (scanner.find(pos)) { + String value = scanner.group(2); + if (value != null) { + if (ret == null) { + ret = new ArrayList(); } - else { - break; - } // we've matched the ']', it's group(4) + ret.add(value); + pos = scanner.end(); } - if (did_nothing) return null; + else { + break; + } // we've matched the ']', it's group(4) } } - catch (IOException e) { - LOG.error("Failed to read script", e); - return null; - } return ret; } + /** + * Extracts paths from site.py generated by buildout 1.5+ + * + * @param vFile path to site.py + * @return extracted paths + */ + public static List extractFromSitePy(VirtualFile vFile) throws IOException { + List result = new ArrayList(); + String text = VfsUtil.loadText(vFile); + String[] lines = LineTokenizer.tokenize(text, false); + int index = 0; + while(index < lines.length && !lines [index].startsWith("def addsitepackages(")) + index++; + while(index < lines.length && !lines [index].trim().startsWith("buildout_paths = [")) + index++; + index++; + while(index < lines.length && !lines [index].trim().equals("]")) { + String line = lines [index].trim(); + if (line.endsWith(",")) { + line = line.substring(0, line.length()-1); + } + if (line.startsWith("'") && line.endsWith("'")) { + result.add(StringUtil.unescapeStringCharacters(line.substring(1, line.length()-1))); + } + index++; + } + return result; + } + @Override public List getAdditionalPythonPath() { BuildoutFacetConfiguration cfg = getConfiguration(); @@ -261,7 +308,7 @@ public class BuildoutFacet extends Facet implements @Nullable public static File findScript(Project project, @Nullable BuildoutFacet buildoutFacet, String name) { - String scriptName = SystemInfo.isWindows ? name + "-script" : name; + String scriptName = SystemInfo.isWindows ? name + SCRIPT_SUFFIX : name; final List scripts = getScripts(project, buildoutFacet); for (File script : scripts) { if (FileUtil.getNameWithoutExtension(script.getName()).equals(scriptName)) { diff --git a/python/src/com/jetbrains/python/console/PyConsoleIndentUtil.java b/python/src/com/jetbrains/python/console/PyConsoleIndentUtil.java index 7728d30fc331..e64c60806c01 100644 --- a/python/src/com/jetbrains/python/console/PyConsoleIndentUtil.java +++ b/python/src/com/jetbrains/python/console/PyConsoleIndentUtil.java @@ -23,11 +23,9 @@ public class PyConsoleIndentUtil { List lines = Lists.newArrayList(); List indents = Lists.newArrayList(); - int minIndent = Integer.MAX_VALUE; while (s.hasNextLine()) { String line = s.nextLine(); int indent = 0; - boolean flag = false; for (char c : line.toCharArray()) { if (c == ' ') { indent++; @@ -36,15 +34,13 @@ public class PyConsoleIndentUtil { indent += TAB_INDENT; } else { - flag = true; break; } } - if (flag && indent < minIndent) { - minIndent = indent; + if (!StringUtil.isEmpty(line)) { + lines.add(line.trim()); + indents.add(indent); } - lines.add(line.trim()); - indents.add(indent); } int[] indentArray = ArrayUtil.toIntArray(indents); @@ -105,7 +101,6 @@ public class PyConsoleIndentUtil { else { lastIndented = false; } - } } } diff --git a/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java b/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java index 2db11e83be75..b6b1428490e6 100644 --- a/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java +++ b/python/src/com/jetbrains/python/console/PydevConsoleExecuteActionHandler.java @@ -57,7 +57,7 @@ public class PydevConsoleExecuteActionHandler extends ConsoleExecuteActionHandle while (s.hasNextLine()) { String line = s.nextLine(); int indentSize = myIndentHelper.getIndent(line, false); - if (indentSize < myCurrentIndentSize &&!shouldIndent(line)) { + if (indentSize == 0 && indentSize < myCurrentIndentSize &&!shouldIndent(line)) { doProcessLine("\n"); doProcessLine(line); } diff --git a/python/src/com/jetbrains/python/console/RunPythonConsoleAction.java b/python/src/com/jetbrains/python/console/RunPythonConsoleAction.java index 51f740048ec1..df75cd28d2de 100644 --- a/python/src/com/jetbrains/python/console/RunPythonConsoleAction.java +++ b/python/src/com/jetbrains/python/console/RunPythonConsoleAction.java @@ -13,6 +13,7 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.roots.ModuleRootManager; import com.intellij.openapi.util.IconLoader; +import com.intellij.openapi.util.io.FileUtil; import com.jetbrains.python.PyBundle; import com.jetbrains.python.buildout.BuildoutFacet; import com.jetbrains.python.run.PythonCommandLineState; @@ -71,14 +72,9 @@ public class RunPythonConsoleAction extends AnAction implements DumbAware { Collection pythonPath = PythonCommandLineState.collectPythonPath(module); - final String path = Joiner.on(", ").join(Collections2.transform(pythonPath, new Function() { - @Override - public String apply(String input) { - return "'" + input + "'"; - } - })); + final String self_path_append = constructPythonPathCommand(pythonPath); + String workingDir = ModuleRootManager.getInstance(module).getContentRoots()[0].getPath(); - final String self_path_append = "sys.path.extend([" + path + "])"; BuildoutFacet facet = BuildoutFacet.getInstance(module); if (facet != null) { setup_fragment = new String[]{facet.getPathPrependStatement(), self_path_append}; @@ -89,4 +85,15 @@ public class RunPythonConsoleAction extends AnAction implements DumbAware { return PydevConsoleRunner.run(project, sdk, PyBundle.message("python.console"), workingDir, setup_fragment); } + + public static String constructPythonPathCommand(Collection pythonPath) { + final String path = Joiner.on(", ").join(Collections2.transform(pythonPath, new Function() { + @Override + public String apply(String input) { + return "'" + FileUtil.toSystemDependentName(input).replace("\\", "\\\\") + "'"; + } + })); + + return "sys.path.extend([" + path + "])"; + } } diff --git a/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java b/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java index dd1ad43509d9..16e6b89b16d6 100644 --- a/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java @@ -338,9 +338,11 @@ public class PyUnresolvedReferencesInspection extends PyInspection { if (severity == HighlightSeverity.WARNING) { hl_type = ProblemHighlightType.GENERIC_ERROR_OR_WARNING; } + /* else if (severity == HighlightSeverity.ERROR) { hl_type = ProblemHighlightType.ERROR; } + */ else { hl_type = ProblemHighlightType.LIKE_UNKNOWN_SYMBOL; } diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkType.java b/python/src/com/jetbrains/python/sdk/PythonSdkType.java index 58a579c463be..97791955cc81 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkType.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkType.java @@ -283,6 +283,7 @@ public class PythonSdkType extends SdkType { return name; } + @Nullable public AdditionalDataConfigurable createAdditionalDataConfigurable(final SdkModel sdkModel, final SdkModificator sdkModificator) { return null; } @@ -493,7 +494,6 @@ public class PythonSdkType extends SdkType { if (SystemInfo.isLinux) { final VirtualFile file = LocalFileSystem.getInstance().findFileByPath("/usr/lib/python-django"); if (file != null){ - sdkModificator.addRoot(file, OrderRootType.SOURCES); sdkModificator.addRoot(file, OrderRootType.CLASSES); } } @@ -509,7 +509,9 @@ public class PythonSdkType extends SdkType { child = JarFileSystem.getInstance().getJarRootForLocalFile(child); } if (child != null) { - sdkModificator.addRoot(child, OrderRootType.SOURCES); + // NOTE: Files marked as library sources are not considered part of project source. Since the directory of the project the + // user is working on is included in PYTHONPATH with many configurations (e.g. virtualenv), we must not mark SDK paths as + // library sources, only as classes. sdkModificator.addRoot(child, OrderRootType.CLASSES); } } diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java index 35d4356d9080..be15363f75d4 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java @@ -19,6 +19,8 @@ import org.jetbrains.annotations.NotNull; import java.io.*; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; /** @@ -101,24 +103,26 @@ public class PythonSdkUpdater implements ProjectComponent { } private static void updateSdkPath(Sdk sdk, List sysPath) { - // HACK: SDK roots configured by user are added as roots of type CLASSES only, and roots configured from sys.path - // are both classes and sources - final VirtualFile[] oldClassesRoots = sdk.getRootProvider().getFiles(OrderRootType.CLASSES); - final VirtualFile[] oldSourcesRoots = sdk.getRootProvider().getFiles(OrderRootType.SOURCES); + final List oldRoots = Arrays.asList(sdk.getRootProvider().getFiles(OrderRootType.CLASSES)); + final VirtualFile[] sourceRoots = sdk.getRootProvider().getFiles(OrderRootType.SOURCES); + PythonSdkAdditionalData additionalData = sdk.getSdkAdditionalData() instanceof PythonSdkAdditionalData + ? (PythonSdkAdditionalData) sdk.getSdkAdditionalData() + : null; List newRoots = new ArrayList(); for(String root: sysPath) { if (new File(root).exists() && !"egg-info".equals(FileUtil.getExtension(root)) && - !wasOldRoot(root, oldClassesRoots) && - !wasOldRoot(root, oldSourcesRoots)) { + (additionalData == null || !wasOldRoot(root, additionalData.getExcludedPaths())) && + !wasOldRoot(root, oldRoots)) { newRoots.add(root); } } - if (!newRoots.isEmpty()) { + if (!newRoots.isEmpty() || sourceRoots.length > 0) { final SdkModificator modificator = sdk.getSdkModificator(); for (String root : newRoots) { PythonSdkType.addSdkRoot(modificator, root); } + modificator.removeRoots(OrderRootType.SOURCES); ApplicationManager.getApplication().runWriteAction(new Runnable() { @Override public void run() { @@ -128,7 +132,7 @@ public class PythonSdkUpdater implements ProjectComponent { } } - private static boolean wasOldRoot(String root, VirtualFile[] virtualFiles) { + private static boolean wasOldRoot(String root, Collection virtualFiles) { String rootPath = canonicalize(root); for (VirtualFile virtualFile : virtualFiles) { if (canonicalize(virtualFile.getPath()).equals(rootPath)) { diff --git a/python/testData/buildout/django-script.py b/python/testData/buildout/django-script.py new file mode 100644 index 000000000000..e390771eb7dc --- /dev/null +++ b/python/testData/buildout/django-script.py @@ -0,0 +1,18 @@ +#!"C:\Python26\python.exe" + +import sys +sys.path[0:0] = [ + 'c:\\src\\django\\buildout15\\src', + 'c:\\src\\django\\buildout15\\eggs\\djangorecipe-0.20-py2.6.egg', + 'c:\\src\\django\\buildout15\\eggs\\zc.recipe.egg-1.3.2-py2.6.egg', + 'c:\\src\\django\\buildout15\\eggs\\zc.buildout-1.5.2-py2.6.egg', + 'c:\\src\\django\\buildout15\\eggs\\setuptools-0.6c12dev_r88124-py2.6.egg', + 'c:\\src\\django\\buildout15\\parts\\django', + 'c:\\src\\django\\buildout15', + ] + + +import djangorecipe.manage + +if __name__ == '__main__': + djangorecipe.manage.main('shorturls.testsettings') diff --git a/python/testData/buildout/site.py b/python/testData/buildout/site.py new file mode 100644 index 000000000000..fa7b2a4707be --- /dev/null +++ b/python/testData/buildout/site.py @@ -0,0 +1,604 @@ +"""Append module search paths for third-party packages to sys.path. + +**************************************************************** +* This module is automatically imported during initialization. * +**************************************************************** + +In earlier versions of Python (up to 1.5a3), scripts or modules that +needed to use site-specific modules would place ``import site'' +somewhere near the top of their code. Because of the automatic +import, this is no longer necessary (but code that does it still +works). + +This will append site-specific paths to the module search path. On +Unix (including Mac OSX), it starts with sys.prefix and +sys.exec_prefix (if different) and appends +lib/python/site-packages as well as lib/site-python. +On other platforms (such as Windows), it tries each of the +prefixes directly, as well as with lib/site-packages appended. The +resulting directories, if they exist, are appended to sys.path, and +also inspected for path configuration files. + +A path configuration file is a file whose name has the form +.pth; its contents are additional directories (one per line) +to be added to sys.path. Non-existing directories (or +non-directories) are never added to sys.path; no directory is added to +sys.path more than once. Blank lines and lines beginning with +'#' are skipped. Lines starting with 'import' are executed. + +For example, suppose sys.prefix and sys.exec_prefix are set to +/usr/local and there is a directory /usr/local/lib/python2.5/site-packages +with three subdirectories, foo, bar and spam, and two path +configuration files, foo.pth and bar.pth. Assume foo.pth contains the +following: + + # foo package configuration + foo + bar + bletch + +and bar.pth contains: + + # bar package configuration + bar + +Then the following directories are added to sys.path, in this order: + + /usr/local/lib/python2.5/site-packages/bar + /usr/local/lib/python2.5/site-packages/foo + +Note that bletch is omitted because it doesn't exist; bar precedes foo +because bar.pth comes alphabetically before foo.pth; and spam is +omitted because it is not mentioned in either path configuration file. + +After these path manipulations, an attempt is made to import a module +named sitecustomize, which can perform arbitrary additional +site-specific customizations. If this import fails with an +ImportError exception, it is silently ignored. + +""" + +import sys +import os +import __builtin__ + +# Prefixes for site-packages; add additional prefixes like /usr/local here +PREFIXES = [sys.prefix, sys.exec_prefix] +# Enable per user site-packages directory +# set it to False to disable the feature or True to force the feature +ENABLE_USER_SITE = False # buildout does not support user sites. +# for distutils.commands.install +USER_SITE = None +USER_BASE = None + + +def makepath(*paths): + dir = os.path.abspath(os.path.join(*paths)) + return dir, os.path.normcase(dir) + + +def abs__file__(): + """Set all module' __file__ attribute to an absolute path""" + for m in sys.modules.values(): + if hasattr(m, '__loader__'): + continue # don't mess with a PEP 302-supplied __file__ + try: + m.__file__ = os.path.abspath(m.__file__) + except AttributeError: + continue + + +def removeduppaths(): + """ Remove duplicate entries from sys.path along with making them + absolute""" + # This ensures that the initial path provided by the interpreter contains + # only absolute pathnames, even if we're running from the build directory. + L = [] + known_paths = set() + for dir in sys.path: + # Filter out duplicate paths (on case-insensitive file systems also + # if they only differ in case); turn relative paths into absolute + # paths. + dir, dircase = makepath(dir) + if not dircase in known_paths: + L.append(dir) + known_paths.add(dircase) + sys.path[:] = L + return known_paths + +# XXX This should not be part of site.py, since it is needed even when +# using the -S option for Python. See http://www.python.org/sf/586680 +def addbuilddir(): + """Append ./build/lib. in case we're running in the build dir + (especially for Guido :-)""" + from distutils.util import get_platform + s = "build/lib.%s-%.3s" % (get_platform(), sys.version) + if hasattr(sys, 'gettotalrefcount'): + s += '-pydebug' + s = os.path.join(os.path.dirname(sys.path[-1]), s) + sys.path.append(s) + + +def _init_pathinfo(): + """Return a set containing all existing directory entries from sys.path""" + d = set() + for dir in sys.path: + try: + if os.path.isdir(dir): + dir, dircase = makepath(dir) + d.add(dircase) + except TypeError: + continue + return d + + +def addpackage(sitedir, name, known_paths): + """Process a .pth file within the site-packages directory: + For each line in the file, either combine it with sitedir to a path + and add that to known_paths, or execute it if it starts with 'import '. + """ + if known_paths is None: + _init_pathinfo() + reset = 1 + else: + reset = 0 + fullname = os.path.join(sitedir, name) + try: + f = open(fullname, "rU") + except IOError: + return + with f: + for line in f: + if line.startswith("#"): + continue + if line.startswith(("import ", "import\t")): + exec line + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + if reset: + known_paths = None + return known_paths + + +def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" + if known_paths is None: + known_paths = _init_pathinfo() + reset = 1 + else: + reset = 0 + sitedir, sitedircase = makepath(sitedir) + if not sitedircase in known_paths: + sys.path.append(sitedir) # Add path component + try: + names = os.listdir(sitedir) + except os.error: + return + dotpth = os.extsep + "pth" + names = [name for name in names if name.endswith(dotpth)] + for name in sorted(names): + addpackage(sitedir, name, known_paths) + if reset: + known_paths = None + return known_paths + + +def check_enableusersite(): + """Check if user site directory is safe for inclusion + + The function tests for the command line flag (including environment var), + process uid/gid equal to effective uid/gid. + + None: Disabled for security reasons + False: Disabled by user (command line option) + True: Safe and enabled + """ + if sys.flags.no_user_site: + return False + + if hasattr(os, "getuid") and hasattr(os, "geteuid"): + # check process uid == effective uid + if os.geteuid() != os.getuid(): + return None + if hasattr(os, "getgid") and hasattr(os, "getegid"): + # check process gid == effective gid + if os.getegid() != os.getgid(): + return None + + return True + + +def addusersitepackages(known_paths): + """Add a per user site-package to sys.path + + Each user has its own python directory with site-packages in the + home directory. + + USER_BASE is the root directory for all Python versions + + USER_SITE is the user specific site-packages directory + + USER_SITE/.. can be used for data. + """ + global USER_BASE, USER_SITE, ENABLE_USER_SITE + env_base = os.environ.get("PYTHONUSERBASE", None) + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + #if sys.platform in ('os2emx', 'riscos'): + # # Don't know what to put here + # USER_BASE = '' + # USER_SITE = '' + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + USER_BASE = env_base if env_base else joinuser(base, "Python") + USER_SITE = os.path.join(USER_BASE, + "Python" + sys.version[0] + sys.version[2], + "site-packages") + else: + USER_BASE = env_base if env_base else joinuser("~", ".local") + USER_SITE = os.path.join(USER_BASE, "lib", + "python" + sys.version[:3], + "site-packages") + + if ENABLE_USER_SITE and os.path.isdir(USER_SITE): + addsitedir(USER_SITE, known_paths) + return known_paths + + +def addsitepackages(known_paths): + """Add site packages, as determined by zc.buildout. + + See original_addsitepackages, below, for the original version.""" + setuptools_path = 'c:\\src\\django\\buildout15\\eggs\\setuptools-0.6c12dev_r88124-py2.6.egg' + sys.path.append(setuptools_path) + known_paths.add(os.path.normcase(setuptools_path)) + import pkg_resources + buildout_paths = [ + 'c:\\src\\django\\buildout15\\src', + 'c:\\src\\django\\buildout15\\eggs\\setuptools-0.6c12dev_r88124-py2.6.egg' + ] + for path in buildout_paths: + sitedir, sitedircase = makepath(path) + if not sitedircase in known_paths and os.path.exists(sitedir): + sys.path.append(sitedir) + known_paths.add(sitedircase) + pkg_resources.working_set.add_entry(sitedir) + sys.__egginsert = len(buildout_paths) # Support distribute. + original_paths = [ + 'C:\\Python26\\lib\\site-packages' + ] + for path in original_paths: + if path == setuptools_path or path not in known_paths: + addsitedir(path, known_paths) + return known_paths + +def original_addsitepackages(known_paths): + """Add site-packages (and possibly site-python) to sys.path""" + sitedirs = [] + seen = [] + + for prefix in PREFIXES: + if not prefix or prefix in seen: + continue + seen.append(prefix) + + if sys.platform in ('os2emx', 'riscos'): + sitedirs.append(os.path.join(prefix, "Lib", "site-packages")) + elif os.sep == '/': + sitedirs.append(os.path.join(prefix, "lib", + "python" + sys.version[:3], + "site-packages")) + sitedirs.append(os.path.join(prefix, "lib", "site-python")) + else: + sitedirs.append(prefix) + sitedirs.append(os.path.join(prefix, "lib", "site-packages")) + + if sys.platform == "darwin": + # for framework builds *only* we add the standard Apple + # locations. Currently only per-user, but /Library and + # /Network/Library could be added too + if 'Python.framework' in prefix: + sitedirs.append( + os.path.expanduser( + os.path.join("~", "Library", "Python", + sys.version[:3], "site-packages"))) + + for sitedir in sitedirs: + if os.path.isdir(sitedir): + addsitedir(sitedir, known_paths) + + return known_paths + + +def setBEGINLIBPATH(): + """The OS/2 EMX port has optional extension modules that do double duty + as DLLs (and must use the .DLL file extension) for other extensions. + The library search path needs to be amended so these will be found + during module import. Use BEGINLIBPATH so that these are at the start + of the library search path. + + """ + dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload") + libpath = os.environ['BEGINLIBPATH'].split(';') + if libpath[-1]: + libpath.append(dllpath) + else: + libpath[-1] = dllpath + os.environ['BEGINLIBPATH'] = ';'.join(libpath) + + +def setquit(): + """Define new built-ins 'quit' and 'exit'. + These are simply strings that display a hint on how to exit. + + """ + if os.sep == ':': + eof = 'Cmd-Q' + elif os.sep == '\\': + eof = 'Ctrl-Z plus Return' + else: + eof = 'Ctrl-D (i.e. EOF)' + + class Quitter(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return 'Use %s() or %s to exit' % (self.name, eof) + def __call__(self, code=None): + # Shells like IDLE catch the SystemExit, but listen when their + # stdin wrapper is closed. + try: + sys.stdin.close() + except: + pass + raise SystemExit(code) + __builtin__.quit = Quitter('quit') + __builtin__.exit = Quitter('exit') + + +class _Printer(object): + """interactive prompt objects for printing the license text, a list of + contributors and the copyright notice.""" + + MAXLINES = 23 + + def __init__(self, name, data, files=(), dirs=()): + self.__name = name + self.__data = data + self.__files = files + self.__dirs = dirs + self.__lines = None + + def __setup(self): + if self.__lines: + return + data = None + for dir in self.__dirs: + for filename in self.__files: + filename = os.path.join(dir, filename) + try: + fp = file(filename, "rU") + data = fp.read() + fp.close() + break + except IOError: + pass + if data: + break + if not data: + data = self.__data + self.__lines = data.split('\n') + self.__linecnt = len(self.__lines) + + def __repr__(self): + self.__setup() + if len(self.__lines) <= self.MAXLINES: + return "\n".join(self.__lines) + else: + return "Type %s() to see the full %s text" % ((self.__name,)*2) + + def __call__(self): + self.__setup() + prompt = 'Hit Return for more, or q (and Return) to quit: ' + lineno = 0 + while 1: + try: + for i in range(lineno, lineno + self.MAXLINES): + print self.__lines[i] + except IndexError: + break + else: + lineno += self.MAXLINES + key = None + while key is None: + key = raw_input(prompt) + if key not in ('', 'q'): + key = None + if key == 'q': + break + +def setcopyright(): + """Set 'copyright' and 'credits' in __builtin__""" + __builtin__.copyright = _Printer("copyright", sys.copyright) + if sys.platform[:4] == 'java': + __builtin__.credits = _Printer( + "credits", + "Jython is maintained by the Jython developers (www.jython.org).") + else: + __builtin__.credits = _Printer("credits", """\ + Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands + for supporting Python development. See www.python.org for more information.""") + here = os.path.dirname(os.__file__) + __builtin__.license = _Printer( + "license", "See http://www.python.org/%.3s/license.html" % sys.version, + ["LICENSE.txt", "LICENSE"], + [os.path.join(here, os.pardir), here, os.curdir]) + + +class _Helper(object): + """Define the built-in 'help'. + This is a wrapper around pydoc.help (with a twist). + + """ + + def __repr__(self): + return "Type help() for interactive help, " \ + "or help(object) for help about object." + def __call__(self, *args, **kwds): + import pydoc + return pydoc.help(*args, **kwds) + +def sethelper(): + __builtin__.help = _Helper() + +def aliasmbcs(): + """On Windows, some default encodings are not provided by Python, + while they are always available as "mbcs" in each locale. Make + them usable by aliasing to "mbcs" in such a case.""" + if sys.platform == 'win32': + import locale, codecs + enc = locale.getdefaultlocale()[1] + if enc.startswith('cp'): # "cp***" ? + try: + codecs.lookup(enc) + except LookupError: + import encodings + encodings._cache[enc] = encodings._unknown + encodings.aliases.aliases[enc] = 'mbcs' + +def setencoding(): + """Set the string encoding used by the Unicode implementation. The + default is 'ascii', but if you're willing to experiment, you can + change this.""" + encoding = "ascii" # Default value set by _PyUnicode_Init() + if 0: + # Enable to support locale aware default string encodings. + import locale + loc = locale.getdefaultlocale() + if loc[1]: + encoding = loc[1] + if 0: + # Enable to switch off string to Unicode coercion and implicit + # Unicode to string conversion. + encoding = "undefined" + if encoding != "ascii": + # On Non-Unicode builds this will raise an AttributeError... + sys.setdefaultencoding(encoding) # Needs Python Unicode build ! + + +def execsitecustomize(): + """Run custom site specific code, if available.""" + try: + import sitecustomize + except ImportError: + pass + except Exception: + if sys.flags.verbose: + sys.excepthook(*sys.exc_info()) + else: + print >>sys.stderr, \ + "'import sitecustomize' failed; use -v for traceback" + + +def execusercustomize(): + """Run custom user specific code, if available.""" + try: + import usercustomize + except ImportError: + pass + except Exception: + if sys.flags.verbose: + sys.excepthook(*sys.exc_info()) + else: + print>>sys.stderr, \ + "'import usercustomize' failed; use -v for traceback" + + +def main(): + global ENABLE_USER_SITE + + abs__file__() + known_paths = removeduppaths() + if (os.name == "posix" and sys.path and + os.path.basename(sys.path[-1]) == "Modules"): + addbuilddir() + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + known_paths = addusersitepackages(known_paths) + known_paths = addsitepackages(known_paths) + if sys.platform == 'os2emx': + setBEGINLIBPATH() + setquit() + setcopyright() + sethelper() + aliasmbcs() + setencoding() + execsitecustomize() + if ENABLE_USER_SITE: + execusercustomize() + # Remove sys.setdefaultencoding() so that users cannot change the + # encoding after initialization. The test for presence is needed when + # this module is run as a script, because this code is executed twice. + if hasattr(sys, "setdefaultencoding"): + del sys.setdefaultencoding + +main() + +def _script(): + help = """\ + %s [--user-base] [--user-site] + + Without arguments print some useful information + With arguments print the value of USER_BASE and/or USER_SITE separated + by '%s'. + + Exit codes with --user-base or --user-site: + 0 - user site directory is enabled + 1 - user site directory is disabled by user + 2 - uses site directory is disabled by super user + or for security reasons + >2 - unknown error + """ + args = sys.argv[1:] + if not args: + print "sys.path = [" + for dir in sys.path: + print " %r," % (dir,) + print "]" + print "USER_BASE: %r (%s)" % (USER_BASE, + "exists" if os.path.isdir(USER_BASE) else "doesn't exist") + print "USER_SITE: %r (%s)" % (USER_SITE, + "exists" if os.path.isdir(USER_SITE) else "doesn't exist") + print "ENABLE_USER_SITE: %r" % ENABLE_USER_SITE + sys.exit(0) + + buffer = [] + if '--user-base' in args: + buffer.append(USER_BASE) + if '--user-site' in args: + buffer.append(USER_SITE) + + if buffer: + print os.pathsep.join(buffer) + if ENABLE_USER_SITE: + sys.exit(0) + elif ENABLE_USER_SITE is False: + sys.exit(1) + elif ENABLE_USER_SITE is None: + sys.exit(2) + else: + sys.exit(3) + else: + import textwrap + print textwrap.dedent(help % (sys.argv[0], os.pathsep)) + sys.exit(10) + +if __name__ == '__main__': + _script() diff --git a/python/testData/inspections/AddClass.py b/python/testData/inspections/AddClass.py index bbbe7146c931..d1e7363de71a 100644 --- a/python/testData/inspections/AddClass.py +++ b/python/testData/inspections/AddClass.py @@ -1,3 +1,3 @@ class XyzzyTest: def testSimple(self): - c = Xyzzy() + c = Xyzzy() diff --git a/python/testData/inspections/AddImport.py b/python/testData/inspections/AddImport.py index 76ef6bdfefe1..b8a569b74c6f 100644 --- a/python/testData/inspections/AddImport.py +++ b/python/testData/inspections/AddImport.py @@ -1,3 +1,3 @@ -ImportTarget.foo +ImportTarget.foo foo = 1 diff --git a/python/testData/inspections/AddImportDoc.py b/python/testData/inspections/AddImportDoc.py index 62b1eb46c0e9..d74d8ad4c8b9 100644 --- a/python/testData/inspections/AddImportDoc.py +++ b/python/testData/inspections/AddImportDoc.py @@ -1,4 +1,4 @@ "Doc comment stays intact" -ImportTarget.foo +ImportTarget.foo foo = 1 diff --git a/python/testData/inspections/AddImportDocComment.py b/python/testData/inspections/AddImportDocComment.py index c3ffeacc7723..053fea996f59 100644 --- a/python/testData/inspections/AddImportDocComment.py +++ b/python/testData/inspections/AddImportDocComment.py @@ -1,5 +1,5 @@ # Comments here "Doc comment stays intact" -ImportTarget.foo +ImportTarget.foo foo = 1 diff --git a/python/testData/inspections/AddToImportFromList.py b/python/testData/inspections/AddToImportFromList.py index f1039c8cda68..373ff05ff3af 100644 --- a/python/testData/inspections/AddToImportFromList.py +++ b/python/testData/inspections/AddToImportFromList.py @@ -1,3 +1,3 @@ from AddToImportFromFoo import bar -foo # must get imported +foo # must get imported diff --git a/python/testData/inspections/QualifyByImport.py b/python/testData/inspections/QualifyByImport.py index 407935b1c3dd..70e969c8998f 100644 --- a/python/testData/inspections/QualifyByImport.py +++ b/python/testData/inspections/QualifyByImport.py @@ -1,3 +1,3 @@ import QualifyByImportFoo -foo # must be qualified +foo # must be qualified diff --git a/python/testData/inspections/UnresolvedRefCreateFunction.py b/python/testData/inspections/UnresolvedRefCreateFunction.py index d45ac6d58f26..6bb775243d25 100644 --- a/python/testData/inspections/UnresolvedRefCreateFunction.py +++ b/python/testData/inspections/UnresolvedRefCreateFunction.py @@ -1,2 +1,2 @@ s = 3 -ref(s, t=1) \ No newline at end of file +ref(s, t=1) \ No newline at end of file diff --git a/python/testData/inspections/UnresolvedRefTrueFalse.py b/python/testData/inspections/UnresolvedRefTrueFalse.py index 6daaf5d5bc66..518fa3d68f59 100644 --- a/python/testData/inspections/UnresolvedRefTrueFalse.py +++ b/python/testData/inspections/UnresolvedRefTrueFalse.py @@ -1 +1 @@ -a = true \ No newline at end of file +a = true \ No newline at end of file diff --git a/python/testData/inspections/UnresolvedWith.py b/python/testData/inspections/UnresolvedWith.py index 2e483ccf6c63..135ac77300fa 100644 --- a/python/testData/inspections/UnresolvedWith.py +++ b/python/testData/inspections/UnresolvedWith.py @@ -1,2 +1,2 @@ -with open("x.txt") as f: - data = f.read() \ No newline at end of file +with open("x.txt") as f: + data = f.read() \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/buildout/BuildoutScriptParserTest.java b/python/testSrc/com/jetbrains/python/buildout/BuildoutScriptParserTest.java new file mode 100644 index 000000000000..0f7ade1c9a88 --- /dev/null +++ b/python/testSrc/com/jetbrains/python/buildout/BuildoutScriptParserTest.java @@ -0,0 +1,32 @@ +package com.jetbrains.python.buildout; + +import com.intellij.openapi.vfs.LocalFileSystem; +import com.intellij.openapi.vfs.VirtualFile; +import com.jetbrains.python.fixtures.PyLightFixtureTestCase; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * @author yole + */ +public class BuildoutScriptParserTest extends PyLightFixtureTestCase { + public void testParseOldStyleScript() throws IOException { + File scriptFile = new File(myFixture.getTestDataPath(), "buildout/django-script.py"); + VirtualFile vFile = LocalFileSystem.getInstance().findFileByIoFile(scriptFile); + List paths = BuildoutFacet.extractFromScript(vFile); + assertEquals(7, paths.size()); + assertTrue(paths.contains("c:\\\\src\\\\django\\\\buildout15\\\\eggs\\\\djangorecipe-0.20-py2.6.egg")); + assertTrue(paths.contains("c:\\\\src\\\\django\\\\buildout15\\\\parts\\\\django")); + } + + public void testParseSitePy() throws IOException { + File scriptFile = new File(myFixture.getTestDataPath(), "buildout/site.py"); + VirtualFile vFile = LocalFileSystem.getInstance().findFileByIoFile(scriptFile); + List paths = BuildoutFacet.extractFromSitePy(vFile); + assertSameElements(paths, + "c:\\src\\django\\buildout15\\src", + "c:\\src\\django\\buildout15\\eggs\\setuptools-0.6c12dev_r88124-py2.6.egg"); + } +}