From 09705492dd4db6d458caca230a06bbff04b5481d Mon Sep 17 00:00:00 2001 From: Dmitry Trofimov Date: Mon, 14 Jul 2014 12:10:14 +0200 Subject: [PATCH] Coverage.py updated to version 3.7.1 --- python/helpers/coverage/__init__.py | 47 +- python/helpers/coverage/__main__.py | 5 +- python/helpers/coverage/annotate.py | 19 +- python/helpers/coverage/backward.py | 96 ++- python/helpers/coverage/bytecode.py | 74 +- python/helpers/coverage/cmdline.py | 354 +++++---- python/helpers/coverage/codeunit.py | 48 +- python/helpers/coverage/collector.py | 76 +- python/helpers/coverage/config.py | 210 +++-- python/helpers/coverage/control.py | 381 +++++---- python/helpers/coverage/data.py | 52 +- python/helpers/coverage/debug.py | 54 ++ python/helpers/coverage/execfile.py | 90 ++- python/helpers/coverage/files.py | 226 +++++- .../coverage/fullcoverage/encodings.py | 56 ++ python/helpers/coverage/html.py | 179 +++-- .../coverage/htmlfiles/coverage_html.js | 6 +- python/helpers/coverage/htmlfiles/index.html | 13 +- .../{jquery-1.4.3.min.js => jquery.min.js} | 0 .../coverage/htmlfiles/keybd_closed.png | Bin 177 -> 264 bytes .../helpers/coverage/htmlfiles/keybd_open.png | Bin 175 -> 267 bytes python/helpers/coverage/htmlfiles/pyfile.html | 15 +- python/helpers/coverage/htmlfiles/style.css | 29 +- python/helpers/coverage/misc.py | 32 +- python/helpers/coverage/parser.py | 443 +++++------ python/helpers/coverage/phystokens.py | 117 ++- python/helpers/coverage/report.py | 41 +- python/helpers/coverage/results.py | 73 +- python/helpers/coverage/summary.py | 31 +- python/helpers/coverage/templite.py | 226 +++--- python/helpers/coverage/tracer.c | 730 ++++++++++++++++++ python/helpers/coverage/tracer.pyd | Bin 9728 -> 0 bytes python/helpers/coverage/version.py | 9 + python/helpers/coverage/xmlreport.py | 44 +- 34 files changed, 2763 insertions(+), 1013 deletions(-) create mode 100644 python/helpers/coverage/debug.py create mode 100644 python/helpers/coverage/fullcoverage/encodings.py rename python/helpers/coverage/htmlfiles/{jquery-1.4.3.min.js => jquery.min.js} (100%) mode change 100644 => 100755 python/helpers/coverage/htmlfiles/keybd_closed.png mode change 100644 => 100755 python/helpers/coverage/htmlfiles/keybd_open.png create mode 100644 python/helpers/coverage/tracer.c delete mode 100644 python/helpers/coverage/tracer.pyd create mode 100644 python/helpers/coverage/version.py diff --git a/python/helpers/coverage/__init__.py b/python/helpers/coverage/__init__.py index d8dbc0f6dc88..c67518158612 100644 --- a/python/helpers/coverage/__init__.py +++ b/python/helpers/coverage/__init__.py @@ -5,19 +5,13 @@ http://nedbatchelder.com/code/coverage """ -__version__ = "3.5" # see detailed history in CHANGES.txt - -__url__ = "http://nedbatchelder.com/code/coverage" -if max(__version__).isalpha(): - # For pre-releases, use a version-specific URL. - __url__ += "/" + __version__ +from coverage.version import __version__, __url__ from coverage.control import coverage, process_startup from coverage.data import CoverageData from coverage.cmdline import main, CoverageScript from coverage.misc import CoverageException - # Module-level functions. The original API to this module was based on # functions defined directly in the module, with a singleton of the coverage() # class. That design hampered programmability, so the current api uses @@ -36,12 +30,34 @@ def _singleton_method(name): called. """ + # Disable pylint msg W0612, because a bunch of variables look unused, but + # they're accessed via locals(). + # pylint: disable=W0612 + def wrapper(*args, **kwargs): """Singleton wrapper around a coverage method.""" global _the_coverage if not _the_coverage: _the_coverage = coverage(auto_data=True) return getattr(_the_coverage, name)(*args, **kwargs) + + import inspect + meth = getattr(coverage, name) + args, varargs, kw, defaults = inspect.getargspec(meth) + argspec = inspect.formatargspec(args[1:], varargs, kw, defaults) + docstring = meth.__doc__ + wrapper.__doc__ = ("""\ + A first-use-singleton wrapper around coverage.%(name)s. + + This wrapper is provided for backward compatibility with legacy code. + New code should use coverage.%(name)s directly. + + %(name)s%(argspec)s: + + %(docstring)s + """ % locals() + ) + return wrapper @@ -57,10 +73,25 @@ report = _singleton_method('report') annotate = _singleton_method('annotate') +# On Windows, we encode and decode deep enough that something goes wrong and +# the encodings.utf_8 module is loaded and then unloaded, I don't know why. +# Adding a reference here prevents it from being unloaded. Yuk. + +# Because of the "from coverage.control import fooey" lines at the top of the +# file, there's an entry for coverage.coverage in sys.modules, mapped to None. +# This makes some inspection tools (like pydoc) unable to find the class +# coverage.coverage. So remove that entry. +import sys +try: + del sys.modules['coverage.coverage'] +except KeyError: + pass + + # COPYRIGHT AND LICENSE # # Copyright 2001 Gareth Rees. All rights reserved. -# Copyright 2004-2010 Ned Batchelder. All rights reserved. +# Copyright 2004-2013 Ned Batchelder. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/python/helpers/coverage/__main__.py b/python/helpers/coverage/__main__.py index af5fa9f6819f..55e0d259e04a 100644 --- a/python/helpers/coverage/__main__.py +++ b/python/helpers/coverage/__main__.py @@ -1,3 +1,4 @@ -"""Coverage.py's main entrypoint.""" +"""Coverage.py's main entry point.""" +import sys from coverage.cmdline import main -main() +sys.exit(main()) diff --git a/python/helpers/coverage/annotate.py b/python/helpers/coverage/annotate.py index a556d853cdbd..82cb7a7eee1d 100644 --- a/python/helpers/coverage/annotate.py +++ b/python/helpers/coverage/annotate.py @@ -1,7 +1,8 @@ """Source file annotation for Coverage.""" -import os, re - +import os +import re +from coverage.backward import sorted # pylint: disable=W0622 from coverage.report import Reporter class AnnotateReporter(Reporter): @@ -26,20 +27,20 @@ class AnnotateReporter(Reporter): """ - def __init__(self, coverage, ignore_errors=False): - super(AnnotateReporter, self).__init__(coverage, ignore_errors) + def __init__(self, coverage, config): + super(AnnotateReporter, self).__init__(coverage, config) self.directory = None blank_re = re.compile(r"\s*(#|$)") else_re = re.compile(r"\s*else\s*:\s*(#|$)") - def report(self, morfs, config, directory=None): + def report(self, morfs, directory=None): """Run the report. See `coverage.report()` for arguments. """ - self.report_files(self.annotate_file, morfs, config, directory) + self.report_files(self.annotate_file, morfs, directory) def annotate_file(self, cu, analysis): """Annotate a single file. @@ -59,9 +60,9 @@ class AnnotateReporter(Reporter): dest_file = filename + ",cover" dest = open(dest_file, 'w') - statements = analysis.statements - missing = analysis.missing - excluded = analysis.excluded + statements = sorted(analysis.statements) + missing = sorted(analysis.missing) + excluded = sorted(analysis.excluded) lineno = 0 i = 0 diff --git a/python/helpers/coverage/backward.py b/python/helpers/coverage/backward.py index f0a34ac4ceb1..e2a17966a688 100644 --- a/python/helpers/coverage/backward.py +++ b/python/helpers/coverage/backward.py @@ -6,8 +6,6 @@ # W0611: Unused import blah # W0622: Redefining built-in blah -import os, sys - # Python 2.3 doesn't have `set` try: set = set # new in 2.4 @@ -24,6 +22,31 @@ except NameError: lst.sort() return lst +# Python 2.3 doesn't have `reversed`. +try: + reversed = reversed +except NameError: + def reversed(iterable): + """A 2.3-compatible implementation of `reversed`.""" + lst = list(iterable) + return lst[::-1] + +# rpartition is new in 2.5 +try: + "".rpartition +except AttributeError: + def rpartition(s, sep): + """Implement s.rpartition(sep) for old Pythons.""" + i = s.rfind(sep) + if i == -1: + return ('', '', s) + else: + return (s[:i], sep, s[i+len(sep):]) +else: + def rpartition(s, sep): + """A common interface for new Pythons.""" + return s.rpartition(sep) + # Pythons 2 and 3 differ on where to get StringIO try: from cStringIO import StringIO @@ -49,6 +72,18 @@ try: except NameError: range = range +# A function to iterate listlessly over a dict's items. +try: + {}.iteritems +except AttributeError: + def iitems(d): + """Produce the items from dict `d`.""" + return d.items() +else: + def iitems(d): + """Produce the items from dict `d`.""" + return d.iteritems() + # Exec is a statement in Py2, a function in Py3 if sys.version_info >= (3, 0): def exec_code_object(code, global_map): @@ -66,21 +101,32 @@ else: ) ) -# ConfigParser was renamed to the more-standard configparser -try: - import configparser -except ImportError: - import ConfigParser as configparser - -# Python 3.2 provides `tokenize.open`, the best way to open source files. -try: +# Reading Python source and interpreting the coding comment is a big deal. +if sys.version_info >= (3, 0): + # Python 3.2 provides `tokenize.open`, the best way to open source files. import tokenize - open_source = tokenize.open # pylint: disable=E1101 -except AttributeError: + try: + open_source = tokenize.open # pylint: disable=E1101 + except AttributeError: + from io import TextIOWrapper + detect_encoding = tokenize.detect_encoding # pylint: disable=E1101 + # Copied from the 3.2 stdlib: + def open_source(fname): + """Open a file in read only mode using the encoding detected by + detect_encoding(). + """ + buffer = open(fname, 'rb') + encoding, _ = detect_encoding(buffer.readline) + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True) + text.mode = 'r' + return text +else: def open_source(fname): """Open a source file the best way.""" return open(fname, "rU") + # Python 3.x is picky about bytes and strings, so provide methods to # get them right, and make them no-ops in 2.x if sys.version_info >= (3, 0): @@ -92,6 +138,19 @@ if sys.version_info >= (3, 0): """Convert bytes `b` to a string.""" return b.decode('utf8') + def binary_bytes(byte_values): + """Produce a byte string with the ints from `byte_values`.""" + return bytes(byte_values) + + def byte_to_int(byte_value): + """Turn an element of a bytes object into an int.""" + return byte_value + + def bytes_to_ints(bytes_value): + """Turn a bytes object into a sequence of ints.""" + # In Py3, iterating bytes gives ints. + return bytes_value + else: def to_bytes(s): """Convert string `s` to bytes (no-op in 2.x).""" @@ -101,6 +160,19 @@ else: """Convert bytes `b` to a string (no-op in 2.x).""" return b + def binary_bytes(byte_values): + """Produce a byte string with the ints from `byte_values`.""" + return "".join([chr(b) for b in byte_values]) + + def byte_to_int(byte_value): + """Turn an element of a bytes object into an int.""" + return ord(byte_value) + + def bytes_to_ints(bytes_value): + """Turn a bytes object into a sequence of ints.""" + for byte in bytes_value: + yield ord(byte) + # Md5 is available in different places. try: import hashlib diff --git a/python/helpers/coverage/bytecode.py b/python/helpers/coverage/bytecode.py index ab522d6c1c6c..30823db789dd 100644 --- a/python/helpers/coverage/bytecode.py +++ b/python/helpers/coverage/bytecode.py @@ -1,14 +1,25 @@ """Bytecode manipulation for coverage.py""" -import opcode, sys, types +import opcode +import types +from coverage.backward import byte_to_int class ByteCode(object): """A single bytecode.""" def __init__(self): + # The offset of this bytecode in the code object. self.offset = -1 + + # The opcode, defined in the `opcode` module. self.op = -1 + + # The argument, a small integer, whose meaning depends on the opcode. self.arg = -1 + + # The offset in the code object of the next bytecode. self.next_offset = -1 + + # The offset to jump to. self.jump_to = -1 @@ -18,44 +29,34 @@ class ByteCodes(object): Returns `ByteCode` objects. """ + # pylint: disable=R0924 def __init__(self, code): self.code = code - self.offset = 0 - if sys.version_info >= (3, 0): - def __getitem__(self, i): - return self.code[i] - else: - def __getitem__(self, i): - return ord(self.code[i]) + def __getitem__(self, i): + return byte_to_int(self.code[i]) def __iter__(self): - return self + offset = 0 + while offset < len(self.code): + bc = ByteCode() + bc.op = self[offset] + bc.offset = offset - def __next__(self): - if self.offset >= len(self.code): - raise StopIteration + next_offset = offset+1 + if bc.op >= opcode.HAVE_ARGUMENT: + bc.arg = self[offset+1] + 256*self[offset+2] + next_offset += 2 - bc = ByteCode() - bc.op = self[self.offset] - bc.offset = self.offset + label = -1 + if bc.op in opcode.hasjrel: + label = next_offset + bc.arg + elif bc.op in opcode.hasjabs: + label = bc.arg + bc.jump_to = label - next_offset = self.offset+1 - if bc.op >= opcode.HAVE_ARGUMENT: - bc.arg = self[self.offset+1] + 256*self[self.offset+2] - next_offset += 2 - - label = -1 - if bc.op in opcode.hasjrel: - label = next_offset + bc.arg - elif bc.op in opcode.hasjabs: - label = bc.arg - bc.jump_to = label - - bc.next_offset = self.offset = next_offset - return bc - - next = __next__ # Py2k uses an old-style non-dunder name. + bc.next_offset = offset = next_offset + yield bc class CodeObjects(object): @@ -64,18 +65,11 @@ class CodeObjects(object): self.stack = [code] def __iter__(self): - return self - - def __next__(self): - if self.stack: + while self.stack: # We're going to return the code object on the stack, but first # push its children for later returning. code = self.stack.pop() for c in code.co_consts: if isinstance(c, types.CodeType): self.stack.append(c) - return code - - raise StopIteration - - next = __next__ + yield code diff --git a/python/helpers/coverage/cmdline.py b/python/helpers/coverage/cmdline.py index 1ce5e0f54ad6..6cfb05e37b4d 100644 --- a/python/helpers/coverage/cmdline.py +++ b/python/helpers/coverage/cmdline.py @@ -1,10 +1,14 @@ """Command-line support for Coverage.""" -import optparse, re, sys, traceback - +import optparse +import os +import sys +import time +import traceback from coverage.backward import sorted # pylint: disable=W0622 from coverage.execfile import run_python_file, run_python_module from coverage.misc import CoverageException, ExceptionDuringRun, NoSource +from coverage.debug import info_formatter class Opts(object): @@ -19,11 +23,18 @@ class Opts(object): '', '--branch', action='store_true', help="Measure branch coverage in addition to statement coverage." ) + debug = optparse.make_option( + '', '--debug', action='store', metavar="OPTS", + help="Debug options, separated by commas" + ) directory = optparse.make_option( - '-d', '--directory', action='store', - metavar="DIR", + '-d', '--directory', action='store', metavar="DIR", help="Write the output files to DIR." ) + fail_under = optparse.make_option( + '', '--fail-under', action='store', metavar="MIN", type="int", + help="Exit with a status of 2 if the total coverage is less than MIN." + ) help = optparse.make_option( '-h', '--help', action='store_true', help="Get help on this command." @@ -89,6 +100,10 @@ class Opts(object): help="Use a simpler but slower trace method. Try this if you get " "seemingly impossible results!" ) + title = optparse.make_option( + '', '--title', action='store', metavar="TITLE", + help="A text string to use as the title on the HTML." + ) version = optparse.make_option( '', '--version', action='store_true', help="Display version information and exit." @@ -110,7 +125,9 @@ class CoverageOptionParser(optparse.OptionParser, object): self.set_defaults( actions=[], branch=None, + debug=None, directory=None, + fail_under=None, help=None, ignore_errors=None, include=None, @@ -122,6 +139,7 @@ class CoverageOptionParser(optparse.OptionParser, object): show_missing=None, source=None, timid=None, + title=None, erase_first=None, version=None, ) @@ -273,9 +291,11 @@ CMDS = { 'html': CmdOptionParser("html", [ Opts.directory, + Opts.fail_under, Opts.ignore_errors, Opts.omit, Opts.include, + Opts.title, ] + GLOBAL_ARGS, usage = "[options] [modules]", description = "Create an HTML report of the coverage of the files. " @@ -285,6 +305,7 @@ CMDS = { 'report': CmdOptionParser("report", [ + Opts.fail_under, Opts.ignore_errors, Opts.omit, Opts.include, @@ -298,6 +319,7 @@ CMDS = { [ Opts.append, Opts.branch, + Opts.debug, Opts.pylib, Opts.parallel_mode, Opts.module, @@ -314,20 +336,20 @@ CMDS = { 'xml': CmdOptionParser("xml", [ + Opts.fail_under, Opts.ignore_errors, Opts.omit, Opts.include, Opts.output_xml, ] + GLOBAL_ARGS, cmd = "xml", - defaults = {'outfile': 'coverage.xml'}, usage = "[options] [modules]", description = "Generate an XML report of coverage results." ), } -OK, ERR = 0, 1 +OK, ERR, FAIL_UNDER = 0, 1, 2 class CoverageScript(object): @@ -346,27 +368,10 @@ class CoverageScript(object): self.run_python_file = _run_python_file or run_python_file self.run_python_module = _run_python_module or run_python_module self.help_fn = _help_fn or self.help + self.classic = False self.coverage = None - def help(self, error=None, topic=None, parser=None): - """Display an error message, or the named topic.""" - assert error or topic or parser - if error: - print(error) - print("Use 'coverage help' for help.") - elif parser: - print(parser.format_help().strip()) - else: - # Parse out the topic we want from HELP_TOPICS - topic_list = re.split("(?m)^=+ (\w+) =+$", HELP_TOPICS) - topics = dict(zip(topic_list[1::2], topic_list[2::2])) - help_msg = topics.get(topic, '').strip() - if help_msg: - print(help_msg % self.covpkg.__dict__) - else: - print("Don't know topic %r" % topic) - def command_line(self, argv): """The bulk of the command line interface to Coverage. @@ -376,15 +381,14 @@ class CoverageScript(object): """ # Collect the command-line options. - if not argv: self.help_fn(topic='minimum_help') return OK # The command syntax we parse depends on the first argument. Classic # syntax always starts with an option. - classic = argv[0].startswith('-') - if classic: + self.classic = argv[0].startswith('-') + if self.classic: parser = ClassicOptionParser() else: parser = CMDS.get(argv[0]) @@ -398,64 +402,19 @@ class CoverageScript(object): if not ok: return ERR - # Handle help. - if options.help: - if classic: - self.help_fn(topic='help') - else: - self.help_fn(parser=parser) - return OK - - if "help" in options.actions: - if args: - for a in args: - parser = CMDS.get(a) - if parser: - self.help_fn(parser=parser) - else: - self.help_fn(topic=a) - else: - self.help_fn(topic='help') - return OK - - # Handle version. - if options.version: - self.help_fn(topic='version') + # Handle help and version. + if self.do_help(options, args, parser): return OK # Check for conflicts and problems in the options. - for i in ['erase', 'execute']: - for j in ['annotate', 'html', 'report', 'combine']: - if (i in options.actions) and (j in options.actions): - self.help_fn("You can't specify the '%s' and '%s' " - "options at the same time." % (i, j)) - return ERR - - if not options.actions: - self.help_fn( - "You must specify at least one of -e, -x, -c, -r, -a, or -b." - ) - return ERR - args_allowed = ( - 'execute' in options.actions or - 'annotate' in options.actions or - 'html' in options.actions or - 'debug' in options.actions or - 'report' in options.actions or - 'xml' in options.actions - ) - if not args_allowed and args: - self.help_fn("Unexpected arguments: %s" % " ".join(args)) - return ERR - - if 'execute' in options.actions and not args: - self.help_fn("Nothing to do.") + if not self.args_ok(options, args): return ERR # Listify the list options. source = unshell_list(options.source) omit = unshell_list(options.omit) include = unshell_list(options.include) + debug = unshell_list(options.debug) # Do something. self.coverage = self.covpkg.coverage( @@ -467,41 +426,11 @@ class CoverageScript(object): source = source, omit = omit, include = include, + debug = debug, ) if 'debug' in options.actions: - if not args: - self.help_fn("What information would you like: data, sys?") - return ERR - for info in args: - if info == 'sys': - print("-- sys ----------------------------------------") - for label, info in self.coverage.sysinfo(): - if info == []: - info = "-none-" - if isinstance(info, list): - print("%15s:" % label) - for e in info: - print("%15s %s" % ("", e)) - else: - print("%15s: %s" % (label, info)) - elif info == 'data': - print("-- data ---------------------------------------") - self.coverage.load() - print("path: %s" % self.coverage.data.filename) - print("has_arcs: %r" % self.coverage.data.has_arcs()) - summary = self.coverage.data.summary(fullpath=True) - if summary: - filenames = sorted(summary.keys()) - print("\n%d files:" % len(filenames)) - for f in filenames: - print("%s: %d lines" % (f, summary[f])) - else: - print("No data collected") - else: - self.help_fn("Don't know what you mean by %r" % info) - return ERR - return OK + return self.do_debug(args) if 'erase' in options.actions or options.erase_first: self.coverage.erase() @@ -509,22 +438,7 @@ class CoverageScript(object): self.coverage.load() if 'execute' in options.actions: - # Run the script. - self.coverage.start() - code_ran = True - try: - try: - if options.module: - self.run_python_module(args[0], args) - else: - self.run_python_file(args[0], args) - except NoSource: - code_ran = False - raise - finally: - if code_ran: - self.coverage.stop() - self.coverage.save() + self.do_execute(options, args) if 'combine' in options.actions: self.coverage.combine() @@ -539,18 +453,167 @@ class CoverageScript(object): ) if 'report' in options.actions: - self.coverage.report( + total = self.coverage.report( show_missing=options.show_missing, **report_args) if 'annotate' in options.actions: self.coverage.annotate( directory=options.directory, **report_args) if 'html' in options.actions: - self.coverage.html_report( - directory=options.directory, **report_args) + total = self.coverage.html_report( + directory=options.directory, title=options.title, + **report_args) if 'xml' in options.actions: outfile = options.outfile - self.coverage.xml_report(outfile=outfile, **report_args) + total = self.coverage.xml_report(outfile=outfile, **report_args) + if options.fail_under is not None: + if total >= options.fail_under: + return OK + else: + return FAIL_UNDER + else: + return OK + + def help(self, error=None, topic=None, parser=None): + """Display an error message, or the named topic.""" + assert error or topic or parser + if error: + print(error) + print("Use 'coverage help' for help.") + elif parser: + print(parser.format_help().strip()) + else: + help_msg = HELP_TOPICS.get(topic, '').strip() + if help_msg: + print(help_msg % self.covpkg.__dict__) + else: + print("Don't know topic %r" % topic) + + def do_help(self, options, args, parser): + """Deal with help requests. + + Return True if it handled the request, False if not. + + """ + # Handle help. + if options.help: + if self.classic: + self.help_fn(topic='help') + else: + self.help_fn(parser=parser) + return True + + if "help" in options.actions: + if args: + for a in args: + parser = CMDS.get(a) + if parser: + self.help_fn(parser=parser) + else: + self.help_fn(topic=a) + else: + self.help_fn(topic='help') + return True + + # Handle version. + if options.version: + self.help_fn(topic='version') + return True + + return False + + def args_ok(self, options, args): + """Check for conflicts and problems in the options. + + Returns True if everything is ok, or False if not. + + """ + for i in ['erase', 'execute']: + for j in ['annotate', 'html', 'report', 'combine']: + if (i in options.actions) and (j in options.actions): + self.help_fn("You can't specify the '%s' and '%s' " + "options at the same time." % (i, j)) + return False + + if not options.actions: + self.help_fn( + "You must specify at least one of -e, -x, -c, -r, -a, or -b." + ) + return False + args_allowed = ( + 'execute' in options.actions or + 'annotate' in options.actions or + 'html' in options.actions or + 'debug' in options.actions or + 'report' in options.actions or + 'xml' in options.actions + ) + if not args_allowed and args: + self.help_fn("Unexpected arguments: %s" % " ".join(args)) + return False + + if 'execute' in options.actions and not args: + self.help_fn("Nothing to do.") + return False + + return True + + def do_execute(self, options, args): + """Implementation of 'coverage run'.""" + + # Set the first path element properly. + old_path0 = sys.path[0] + + # Run the script. + self.coverage.start() + code_ran = True + try: + try: + if options.module: + sys.path[0] = '' + self.run_python_module(args[0], args) + else: + filename = args[0] + sys.path[0] = os.path.abspath(os.path.dirname(filename)) + self.run_python_file(filename, args) + except NoSource: + code_ran = False + raise + finally: + self.coverage.stop() + if code_ran: + self.coverage.save() + + # Restore the old path + sys.path[0] = old_path0 + + def do_debug(self, args): + """Implementation of 'coverage debug'.""" + + if not args: + self.help_fn("What information would you like: data, sys?") + return ERR + for info in args: + if info == 'sys': + print("-- sys ----------------------------------------") + for line in info_formatter(self.coverage.sysinfo()): + print(" %s" % line) + elif info == 'data': + print("-- data ---------------------------------------") + self.coverage.load() + print("path: %s" % self.coverage.data.filename) + print("has_arcs: %r" % self.coverage.data.has_arcs()) + summary = self.coverage.data.summary(fullpath=True) + if summary: + filenames = sorted(summary.keys()) + print("\n%d files:" % len(filenames)) + for f in filenames: + print("%s: %d lines" % (f, summary[f])) + else: + print("No data collected") + else: + self.help_fn("Don't know what you mean by %r" % info) + return ERR return OK @@ -568,10 +631,10 @@ def unshell_list(s): return s.split(',') -HELP_TOPICS = r""" - -== classic ==================================================================== -Coverage.py version %(__version__)s +HELP_TOPICS = { +# ------------------------- +'classic': +r"""Coverage.py version %(__version__)s Measure, collect, and report on code coverage in Python programs. Usage: @@ -615,8 +678,9 @@ coverage -a [-d DIR] [-i] [-o DIR,...] [FILE1 FILE2 ...] Coverage data is saved in the file .coverage by default. Set the COVERAGE_FILE environment variable to save it somewhere else. - -== help ======================================================================= +""", +# ------------------------- +'help': """\ Coverage.py, version %(__version__)s Measure, collect, and report on code coverage in Python programs. @@ -635,26 +699,32 @@ Commands: Use "coverage help " for detailed help on any command. Use "coverage help classic" for help on older command syntax. For more information, see %(__url__)s - -== minimum_help =============================================================== +""", +# ------------------------- +'minimum_help': """\ Code coverage for Python. Use 'coverage help' for help. - -== version ==================================================================== +""", +# ------------------------- +'version': """\ Coverage.py, version %(__version__)s. %(__url__)s - -""" +""", +} def main(argv=None): - """The main entrypoint to Coverage. + """The main entry point to Coverage. - This is installed as the script entrypoint. + This is installed as the script entry point. """ if argv is None: argv = sys.argv[1:] try: + start = time.clock() status = CoverageScript().command_line(argv) + end = time.clock() + if 0: + print("time: %.3fs" % (end - start)) except ExceptionDuringRun: # An exception was caught while running the product code. The # sys.exc_info() return tuple is packed into an ExceptionDuringRun diff --git a/python/helpers/coverage/codeunit.py b/python/helpers/coverage/codeunit.py index 55f44a240acf..e96e853ca4bc 100644 --- a/python/helpers/coverage/codeunit.py +++ b/python/helpers/coverage/codeunit.py @@ -1,7 +1,7 @@ """Code unit (module) handling for Coverage.""" -import glob, os - +import glob +import os from coverage.backward import open_source, string_class, StringIO from coverage.misc import CoverageException @@ -52,8 +52,10 @@ class CodeUnit(object): else: f = morf # .pyc files should always refer to a .py instead. - if f.endswith('.pyc'): + if f.endswith('.pyc') or f.endswith('.pyo'): f = f[:-1] + elif f.endswith('$py.class'): # Jython + f = f[:-9] + ".py" self.filename = self.file_locator.canonical_filename(f) if hasattr(morf, '__name__'): @@ -77,12 +79,18 @@ class CodeUnit(object): # Annoying comparison operators. Py3k wants __lt__ etc, and Py2k needs all # of them defined. - def __lt__(self, other): return self.name < other.name - def __le__(self, other): return self.name <= other.name - def __eq__(self, other): return self.name == other.name - def __ne__(self, other): return self.name != other.name - def __gt__(self, other): return self.name > other.name - def __ge__(self, other): return self.name >= other.name + def __lt__(self, other): + return self.name < other.name + def __le__(self, other): + return self.name <= other.name + def __eq__(self, other): + return self.name == other.name + def __ne__(self, other): + return self.name != other.name + def __gt__(self, other): + return self.name > other.name + def __ge__(self, other): + return self.name >= other.name def flat_rootname(self): """A base for a flat filename to correspond to this code unit. @@ -113,5 +121,25 @@ class CodeUnit(object): # Couldn't find source. raise CoverageException( - "No source for code %r." % self.filename + "No source for code '%s'." % self.filename ) + + def should_be_python(self): + """Does it seem like this file should contain Python? + + This is used to decide if a file reported as part of the exection of + a program was really likely to have contained Python in the first + place. + + """ + # Get the file extension. + _, ext = os.path.splitext(self.filename) + + # Anything named *.py* should be Python. + if ext.startswith('.py'): + return True + # A file with no extension should be Python. + if not ext: + return True + # Everything else is probably not Python. + return False diff --git a/python/helpers/coverage/collector.py b/python/helpers/coverage/collector.py index 9c40d16c7b19..fae9eb90d29c 100644 --- a/python/helpers/coverage/collector.py +++ b/python/helpers/coverage/collector.py @@ -1,13 +1,26 @@ """Raw data collector for Coverage.""" -import sys, threading +import os +import sys +import threading try: # Use the C extension code when we can, for speed. - from coverage.tracer import Tracer + from coverage.tracer import CTracer # pylint: disable=F0401,E0611 except ImportError: # Couldn't import the C extension, maybe it isn't built. - Tracer = None + if os.getenv('COVERAGE_TEST_TRACER') == 'c': + # During testing, we use the COVERAGE_TEST_TRACER env var to indicate + # that we've fiddled with the environment to test this fallback code. + # If we thought we had a C tracer, but couldn't import it, then exit + # quickly and clearly instead of dribbling confusing errors. I'm using + # sys.exit here instead of an exception because an exception here + # causes all sorts of other noise in unittest. + sys.stderr.write( + "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n" + ) + sys.exit(1) + CTracer = None class PyTracer(object): @@ -40,12 +53,19 @@ class PyTracer(object): self.last_exc_back = None self.last_exc_firstlineno = 0 self.arcs = False + self.thread = None + self.stopped = False def _trace(self, frame, event, arg_unused): """The trace function passed to sys.settrace.""" - #print("trace event: %s %r @%d" % ( - # event, frame.f_code.co_filename, frame.f_lineno)) + if self.stopped: + return + + if 0: + sys.stderr.write("trace event: %s %r @%d\n" % ( + event, frame.f_code.co_filename, frame.f_lineno + )) if self.last_exc_back: if frame == self.last_exc_back: @@ -61,10 +81,11 @@ class PyTracer(object): # in this file. self.data_stack.append((self.cur_file_data, self.last_line)) filename = frame.f_code.co_filename - tracename = self.should_trace_cache.get(filename) - if tracename is None: + if filename not in self.should_trace_cache: tracename = self.should_trace(filename, frame) self.should_trace_cache[filename] = tracename + else: + tracename = self.should_trace_cache[filename] #print("called, stack is %d deep, tracename is %r" % ( # len(self.data_stack), tracename)) if tracename: @@ -105,15 +126,24 @@ class PyTracer(object): Return a Python function suitable for use with sys.settrace(). """ + self.thread = threading.currentThread() sys.settrace(self._trace) return self._trace def stop(self): """Stop this Tracer.""" + self.stopped = True + if self.thread != threading.currentThread(): + # Called on a different thread than started us: we can't unhook + # ourseves, but we've set the flag that we should stop, so we won't + # do any more tracing. + return + if hasattr(sys, "gettrace") and self.warn: if sys.gettrace() != self._trace: msg = "Trace function changed, measurement is likely wrong: %r" - self.warn(msg % sys.gettrace()) + self.warn(msg % (sys.gettrace(),)) + #print("Stopping tracer on %s" % threading.current_thread().ident) sys.settrace(None) def get_stats(self): @@ -146,7 +176,7 @@ class Collector(object): """Create a collector. `should_trace` is a function, taking a filename, and returning a - canonicalized filename, or False depending on whether the file should + canonicalized filename, or None depending on whether the file should be traced or not. If `timid` is true, then a slower simpler trace function will be @@ -173,7 +203,7 @@ class Collector(object): else: # Being fast: use the C Tracer if it is available, else the Python # trace function. - self._trace_class = Tracer or PyTracer + self._trace_class = CTracer or PyTracer def __repr__(self): return "" % id(self) @@ -190,7 +220,7 @@ class Collector(object): # A cache of the results from should_trace, the decision about whether # to trace execution in a file. A dict of filename to (filename or - # False). + # None). self.should_trace_cache = {} # Our active Tracers. @@ -232,9 +262,29 @@ class Collector(object): if self._collectors: self._collectors[-1].pause() self._collectors.append(self) - #print >>sys.stderr, "Started: %r" % self._collectors + #print("Started: %r" % self._collectors, file=sys.stderr) + + # Check to see whether we had a fullcoverage tracer installed. + traces0 = [] + if hasattr(sys, "gettrace"): + fn0 = sys.gettrace() + if fn0: + tracer0 = getattr(fn0, '__self__', None) + if tracer0: + traces0 = getattr(tracer0, 'traces', []) + # Install the tracer on this thread. - self._start_tracer() + fn = self._start_tracer() + + for args in traces0: + (frame, event, arg), lineno = args + try: + fn(frame, event, arg, lineno=lineno) + except TypeError: + raise Exception( + "fullcoverage must be run with the C trace function." + ) + # Install our installation tracer in threading, to jump start other # threads. threading.settrace(self._installation_trace) diff --git a/python/helpers/coverage/config.py b/python/helpers/coverage/config.py index 6b441ddc45fc..0ddb6e22ee21 100644 --- a/python/helpers/coverage/config.py +++ b/python/helpers/coverage/config.py @@ -1,7 +1,79 @@ """Config file for coverage.py""" import os -from coverage.backward import configparser # pylint: disable=W0622 +import re +import sys +from coverage.backward import string_class, iitems + + +# In py3, # ConfigParser was renamed to the more-standard configparser +try: + import configparser # pylint: disable=F0401 +except ImportError: + import ConfigParser as configparser + + +class HandyConfigParser(configparser.RawConfigParser): + """Our specialization of ConfigParser.""" + + def read(self, filename): + """Read a filename as UTF-8 configuration data.""" + kwargs = {} + if sys.version_info >= (3, 2): + kwargs['encoding'] = "utf-8" + return configparser.RawConfigParser.read(self, filename, **kwargs) + + def get(self, *args, **kwargs): + v = configparser.RawConfigParser.get(self, *args, **kwargs) + def dollar_replace(m): + """Called for each $replacement.""" + # Only one of the groups will have matched, just get its text. + word = [w for w in m.groups() if w is not None][0] + if word == "$": + return "$" + else: + return os.environ.get(word, '') + + dollar_pattern = r"""(?x) # Use extended regex syntax + \$(?: # A dollar sign, then + (?P\w+) | # a plain word, + {(?P\w+)} | # or a {-wrapped word, + (?P[$]) # or a dollar sign. + ) + """ + v = re.sub(dollar_pattern, dollar_replace, v) + return v + + def getlist(self, section, option): + """Read a list of strings. + + The value of `section` and `option` is treated as a comma- and newline- + separated list of strings. Each value is stripped of whitespace. + + Returns the list of strings. + + """ + value_list = self.get(section, option) + values = [] + for value_line in value_list.split('\n'): + for value in value_line.split(','): + value = value.strip() + if value: + values.append(value) + return values + + def getlinelist(self, section, option): + """Read a list of full-line strings. + + The value of `section` and `option` is treated as a newline-separated + list of strings. Each value is stripped of whitespace. + + Returns the list of strings. + + """ + value_list = self.get(section, option) + return list(filter(None, value_list.split('\n'))) + # The default line exclusion regexes DEFAULT_EXCLUDE = [ @@ -29,9 +101,12 @@ class CoverageConfig(object): operation of coverage.py. """ - def __init__(self): """Initialize the configuration attributes to their defaults.""" + # Metadata about the config. + self.attempted_config_files = [] + self.config_files = [] + # Defaults for [run] self.branch = False self.cover_pylib = False @@ -39,6 +114,7 @@ class CoverageConfig(object): self.parallel = False self.timid = False self.source = None + self.debug = [] # Defaults for [report] self.exclude_list = DEFAULT_EXCLUDE[:] @@ -48,13 +124,19 @@ class CoverageConfig(object): self.partial_list = DEFAULT_PARTIAL[:] self.partial_always_list = DEFAULT_PARTIAL_ALWAYS[:] self.precision = 0 + self.show_missing = False # Defaults for [html] self.html_dir = "htmlcov" + self.extra_css = None + self.html_title = "Coverage report" # Defaults for [xml] self.xml_output = "coverage.xml" + # Defaults for [paths] + self.paths = {} + def from_environment(self, env_var): """Read configuration from the `env_var` environment variable.""" # Timidity: for nose users, read an environment variable. This is a @@ -64,93 +146,71 @@ class CoverageConfig(object): if env: self.timid = ('--timid' in env) + MUST_BE_LIST = ["omit", "include", "debug"] + def from_args(self, **kwargs): """Read config values from `kwargs`.""" - for k, v in kwargs.items(): + for k, v in iitems(kwargs): if v is not None: + if k in self.MUST_BE_LIST and isinstance(v, string_class): + v = [v] setattr(self, k, v) - def from_file(self, *files): - """Read configuration from .rc files. + def from_file(self, filename): + """Read configuration from a .rc file. - Each argument in `files` is a file name to read. + `filename` is a file name to read. """ - cp = configparser.RawConfigParser() - cp.read(files) + self.attempted_config_files.append(filename) + cp = HandyConfigParser() + files_read = cp.read(filename) + if files_read is not None: # return value changed in 2.4 + self.config_files.extend(files_read) + + for option_spec in self.CONFIG_FILE_OPTIONS: + self.set_attr_from_config_option(cp, *option_spec) + + # [paths] is special + if cp.has_section('paths'): + for option in cp.options('paths'): + self.paths[option] = cp.getlist('paths', option) + + CONFIG_FILE_OPTIONS = [ # [run] - if cp.has_option('run', 'branch'): - self.branch = cp.getboolean('run', 'branch') - if cp.has_option('run', 'cover_pylib'): - self.cover_pylib = cp.getboolean('run', 'cover_pylib') - if cp.has_option('run', 'data_file'): - self.data_file = cp.get('run', 'data_file') - if cp.has_option('run', 'include'): - self.include = self.get_list(cp, 'run', 'include') - if cp.has_option('run', 'omit'): - self.omit = self.get_list(cp, 'run', 'omit') - if cp.has_option('run', 'parallel'): - self.parallel = cp.getboolean('run', 'parallel') - if cp.has_option('run', 'source'): - self.source = self.get_list(cp, 'run', 'source') - if cp.has_option('run', 'timid'): - self.timid = cp.getboolean('run', 'timid') + ('branch', 'run:branch', 'boolean'), + ('cover_pylib', 'run:cover_pylib', 'boolean'), + ('data_file', 'run:data_file'), + ('debug', 'run:debug', 'list'), + ('include', 'run:include', 'list'), + ('omit', 'run:omit', 'list'), + ('parallel', 'run:parallel', 'boolean'), + ('source', 'run:source', 'list'), + ('timid', 'run:timid', 'boolean'), # [report] - if cp.has_option('report', 'exclude_lines'): - self.exclude_list = \ - self.get_line_list(cp, 'report', 'exclude_lines') - if cp.has_option('report', 'ignore_errors'): - self.ignore_errors = cp.getboolean('report', 'ignore_errors') - if cp.has_option('report', 'include'): - self.include = self.get_list(cp, 'report', 'include') - if cp.has_option('report', 'omit'): - self.omit = self.get_list(cp, 'report', 'omit') - if cp.has_option('report', 'partial_branches'): - self.partial_list = \ - self.get_line_list(cp, 'report', 'partial_branches') - if cp.has_option('report', 'partial_branches_always'): - self.partial_always_list = \ - self.get_line_list(cp, 'report', 'partial_branches_always') - if cp.has_option('report', 'precision'): - self.precision = cp.getint('report', 'precision') + ('exclude_list', 'report:exclude_lines', 'linelist'), + ('ignore_errors', 'report:ignore_errors', 'boolean'), + ('include', 'report:include', 'list'), + ('omit', 'report:omit', 'list'), + ('partial_list', 'report:partial_branches', 'linelist'), + ('partial_always_list', 'report:partial_branches_always', 'linelist'), + ('precision', 'report:precision', 'int'), + ('show_missing', 'report:show_missing', 'boolean'), # [html] - if cp.has_option('html', 'directory'): - self.html_dir = cp.get('html', 'directory') + ('html_dir', 'html:directory'), + ('extra_css', 'html:extra_css'), + ('html_title', 'html:title'), # [xml] - if cp.has_option('xml', 'output'): - self.xml_output = cp.get('xml', 'output') - - def get_list(self, cp, section, option): - """Read a list of strings from the ConfigParser `cp`. - - The value of `section` and `option` is treated as a comma- and newline- - separated list of strings. Each value is stripped of whitespace. - - Returns the list of strings. - - """ - value_list = cp.get(section, option) - values = [] - for value_line in value_list.split('\n'): - for value in value_line.split(','): - value = value.strip() - if value: - values.append(value) - return values - - def get_line_list(self, cp, section, option): - """Read a list of full-line strings from the ConfigParser `cp`. - - The value of `section` and `option` is treated as a newline-separated - list of strings. Each value is stripped of whitespace. - - Returns the list of strings. - - """ - value_list = cp.get(section, option) - return list(filter(None, value_list.split('\n'))) + ('xml_output', 'xml:output'), + ] + def set_attr_from_config_option(self, cp, attr, where, type_=''): + """Set an attribute on self if it exists in the ConfigParser.""" + section, option = where.split(":") + if cp.has_option(section, option): + method = getattr(cp, 'get'+type_) + setattr(self, attr, method(section, option)) diff --git a/python/helpers/coverage/control.py b/python/helpers/coverage/control.py index 5ca1ef9538cc..df043584f10f 100644 --- a/python/helpers/coverage/control.py +++ b/python/helpers/coverage/control.py @@ -1,23 +1,36 @@ """Core control stuff for Coverage.""" -import atexit, os, random, socket, sys - +import atexit +import os +import random +import socket +import sys from coverage.annotate import AnnotateReporter -from coverage.backward import string_class +from coverage.backward import string_class, iitems, sorted # pylint: disable=W0622 from coverage.codeunit import code_unit_factory, CodeUnit from coverage.collector import Collector from coverage.config import CoverageConfig from coverage.data import CoverageData +from coverage.debug import DebugControl from coverage.files import FileLocator, TreeMatcher, FnmatchMatcher -from coverage.files import find_python_files +from coverage.files import PathAliases, find_python_files, prep_patterns from coverage.html import HtmlReporter from coverage.misc import CoverageException, bool_or_none, join_regex +from coverage.misc import file_be_gone from coverage.results import Analysis, Numbers from coverage.summary import SummaryReporter from coverage.xmlreport import XmlReporter +# Pypy has some unusual stuff in the "stdlib". Consider those locations +# when deciding where the stdlib is. +try: + import _structseq # pylint: disable=F0401 +except ImportError: + _structseq = None + + class coverage(object): - """Programmatic access to Coverage. + """Programmatic access to coverage.py. To use:: @@ -25,14 +38,15 @@ class coverage(object): cov = coverage() cov.start() - #.. blah blah (run your code) blah blah .. + #.. call your code .. cov.stop() cov.html_report(directory='covhtml') """ def __init__(self, data_file=None, data_suffix=None, cover_pylib=None, auto_data=False, timid=None, branch=None, config_file=True, - source=None, omit=None, include=None): + source=None, omit=None, include=None, debug=None, + debug_file=None): """ `data_file` is the base name of the data file to use, defaulting to ".coverage". `data_suffix` is appended (with a dot) to `data_file` to @@ -67,6 +81,10 @@ class coverage(object): `include` will be measured, files that match `omit` will not. Each will also accept a single string argument. + `debug` is a list of strings indicating what debugging information is + desired. `debug_file` is the file to write debug messages to, + defaulting to stderr. + """ from coverage import __version__ @@ -96,18 +114,16 @@ class coverage(object): self.config.data_file = env_data_file # 4: from constructor arguments: - if isinstance(omit, string_class): - omit = [omit] - if isinstance(include, string_class): - include = [include] self.config.from_args( data_file=data_file, cover_pylib=cover_pylib, timid=timid, branch=branch, parallel=bool_or_none(data_suffix), - source=source, omit=omit, include=include + source=source, omit=omit, include=include, debug=debug, ) + # Create and configure the debugging controller. + self.debug = DebugControl(self.config.debug, debug_file or sys.stderr) + self.auto_data = auto_data - self.atexit_registered = False # _exclude_re is a dict mapping exclusion list names to compiled # regexes. @@ -125,8 +141,8 @@ class coverage(object): else: self.source_pkgs.append(src) - self.omit = self._prep_patterns(self.config.omit) - self.include = self._prep_patterns(self.config.include) + self.omit = prep_patterns(self.config.omit) + self.include = prep_patterns(self.config.include) self.collector = Collector( self._should_trace, timid=self.config.timid, @@ -151,7 +167,8 @@ class coverage(object): # started rather than wherever the process eventually chdir'd to. self.data = CoverageData( basename=self.config.data_file, - collector="coverage v%s" % __version__ + collector="coverage v%s" % __version__, + debug=self.debug, ) # The dirs for files considered "installed with the interpreter". @@ -162,9 +179,9 @@ class coverage(object): # environments (virtualenv, for example), these modules may be # spread across a few locations. Look at all the candidate modules # we've imported, and take all the different ones. - for m in (atexit, os, random, socket): - if hasattr(m, "__file__"): - m_dir = self._canonical_dir(m.__file__) + for m in (atexit, os, random, socket, _structseq): + if m is not None and hasattr(m, "__file__"): + m_dir = self._canonical_dir(m) if m_dir not in self.pylib_dirs: self.pylib_dirs.append(m_dir) @@ -172,63 +189,61 @@ class coverage(object): # where we are. self.cover_dir = self._canonical_dir(__file__) - # The matchers for _should_trace, created when tracing starts. + # The matchers for _should_trace. self.source_match = None self.pylib_match = self.cover_match = None self.include_match = self.omit_match = None - # Only _harvest_data once per measurement cycle. - self._harvested = False - # Set the reporting precision. Numbers.set_precision(self.config.precision) - # When tearing down the coverage object, modules can become None. - # Saving the modules as object attributes avoids problems, but it is - # quite ad-hoc which modules need to be saved and which references - # need to use the object attributes. - self.socket = socket - self.os = os - self.random = random + # Is it ok for no data to be collected? + self._warn_no_data = True + self._warn_unimported_source = True - def _canonical_dir(self, f): - """Return the canonical directory of the file `f`.""" - return os.path.split(self.file_locator.canonical_filename(f))[0] + # State machine variables: + # Have we started collecting and not stopped it? + self._started = False + # Have we measured some data and not harvested it? + self._measured = False + + atexit.register(self._atexit) + + def _canonical_dir(self, morf): + """Return the canonical directory of the module or file `morf`.""" + return os.path.split(CodeUnit(morf, self.file_locator).filename)[0] def _source_for_file(self, filename): """Return the source file for `filename`.""" if not filename.endswith(".py"): if filename[-4:-1] == ".py": filename = filename[:-1] + elif filename.endswith("$py.class"): # jython + filename = filename[:-9] + ".py" return filename - def _should_trace(self, filename, frame): - """Decide whether to trace execution in `filename` + def _should_trace_with_reason(self, filename, frame): + """Decide whether to trace execution in `filename`, with a reason. This function is called from the trace function. As each new file name is encountered, this function determines whether it is traced or not. - Returns a canonicalized filename if it should be traced, False if it - should not. + Returns a pair of values: the first indicates whether the file should + be traced: it's a canonicalized filename if it should be traced, None + if it should not. The second value is a string, the resason for the + decision. """ - if os is None: - return False + if not filename: + # Empty string is pretty useless + return None, "empty string isn't a filename" if filename.startswith('<'): # Lots of non-file execution is represented with artificial # filenames like "", "", or # "". Don't ever trace these executions, since we # can't do anything with the data later anyway. - return False - - if filename.endswith(".html"): - # Jinja and maybe other templating systems compile templates into - # Python code, but use the template filename as the filename in - # the compiled code. Of course, those filenames are useless later - # so don't bother collecting. TODO: How should we really separate - # out good file extensions from bad? - return False + return None, "not a real filename" self._check_for_packages() @@ -248,64 +263,53 @@ class coverage(object): canonical = self.file_locator.canonical_filename(filename) - # If the user specified source, then that's authoritative about what to - # measure. If they didn't, then we have to exclude the stdlib and - # coverage.py directories. + # If the user specified source or include, then that's authoritative + # about the outer bound of what to measure and we don't have to apply + # any canned exclusions. If they didn't, then we have to exclude the + # stdlib and coverage.py directories. if self.source_match: if not self.source_match.match(canonical): - return False + return None, "falls outside the --source trees" + elif self.include_match: + if not self.include_match.match(canonical): + return None, "falls outside the --include trees" else: # If we aren't supposed to trace installed code, then check if this # is near the Python standard library and skip it if so. if self.pylib_match and self.pylib_match.match(canonical): - return False + return None, "is in the stdlib" # We exclude the coverage code itself, since a little of it will be # measured otherwise. if self.cover_match and self.cover_match.match(canonical): - return False + return None, "is part of coverage.py" - # Check the file against the include and omit patterns. - if self.include_match and not self.include_match.match(canonical): - return False + # Check the file against the omit pattern. if self.omit_match and self.omit_match.match(canonical): - return False + return None, "is inside an --omit pattern" + return canonical, "because we love you" + + def _should_trace(self, filename, frame): + """Decide whether to trace execution in `filename`. + + Calls `_should_trace_with_reason`, and returns just the decision. + + """ + canonical, reason = self._should_trace_with_reason(filename, frame) + if self.debug.should('trace'): + if not canonical: + msg = "Not tracing %r: %s" % (filename, reason) + else: + msg = "Tracing %r" % (filename,) + self.debug.write(msg) return canonical - # To log what should_trace returns, change this to "if 1:" - if 0: - _real_should_trace = _should_trace - def _should_trace(self, filename, frame): # pylint: disable=E0102 - """A logging decorator around the real _should_trace function.""" - ret = self._real_should_trace(filename, frame) - print("should_trace: %r -> %r" % (filename, ret)) - return ret - def _warn(self, msg): """Use `msg` as a warning.""" self._warnings.append(msg) sys.stderr.write("Coverage.py warning: %s\n" % msg) - def _prep_patterns(self, patterns): - """Prepare the file patterns for use in a `FnmatchMatcher`. - - If a pattern starts with a wildcard, it is used as a pattern - as-is. If it does not start with a wildcard, then it is made - absolute with the current directory. - - If `patterns` is None, an empty list is returned. - - """ - patterns = patterns or [] - prepped = [] - for p in patterns or []: - if p.startswith("*") or p.startswith("?"): - prepped.append(p) - else: - prepped.append(self.file_locator.abs_file(p)) - return prepped - def _check_for_packages(self): """Update the source_match matcher with latest imported packages.""" # Our self.source_pkgs attribute is a list of package names we want to @@ -325,17 +329,23 @@ class coverage(object): try: pkg_file = mod.__file__ except AttributeError: - self._warn("Module %s has no Python source." % pkg) + pkg_file = None else: d, f = os.path.split(pkg_file) - if f.startswith('__init__.'): + if f.startswith('__init__'): # This is actually a package, return the directory. pkg_file = d else: pkg_file = self._source_for_file(pkg_file) pkg_file = self.file_locator.canonical_filename(pkg_file) + if not os.path.exists(pkg_file): + pkg_file = None + + if pkg_file: self.source.append(pkg_file) self.source_match.add(pkg_file) + else: + self._warn("Module %s has no Python source." % pkg) for pkg in found: self.source_pkgs.remove(pkg) @@ -354,17 +364,21 @@ class coverage(object): self.data.read() def start(self): - """Start measuring code coverage.""" + """Start measuring code coverage. + + Coverage measurement actually occurs in functions called after `start` + is invoked. Statements in the same scope as `start` won't be measured. + + Once you invoke `start`, you must also call `stop` eventually, or your + process might not shut down cleanly. + + """ if self.run_suffix: # Calling start() means we're running code, so use the run_suffix # as the data_suffix when we eventually save the data. self.data_suffix = self.run_suffix if self.auto_data: self.load() - # Save coverage data when Python exits. - if not self.atexit_registered: - atexit.register(self.save) - self.atexit_registered = True # Create the matchers we need for _should_trace if self.source or self.source_pkgs: @@ -379,13 +393,31 @@ class coverage(object): if self.omit: self.omit_match = FnmatchMatcher(self.omit) - self._harvested = False + # The user may want to debug things, show info if desired. + if self.debug.should('config'): + self.debug.write("Configuration values:") + config_info = sorted(self.config.__dict__.items()) + self.debug.write_formatted_info(config_info) + + if self.debug.should('sys'): + self.debug.write("Debugging info:") + self.debug.write_formatted_info(self.sysinfo()) + self.collector.start() + self._started = True + self._measured = True def stop(self): """Stop measuring code coverage.""" + self._started = False self.collector.stop() - self._harvest_data() + + def _atexit(self): + """Clean up on process shutdown.""" + if self._started: + self.stop() + if self.auto_data: + self.save() def erase(self): """Erase previously-collected coverage data. @@ -449,9 +481,15 @@ class coverage(object): # plenty of distinguishing information. We do this here in # `save()` at the last minute so that the pid will be correct even # if the process forks. - data_suffix = "%s.%s.%06d" % ( - self.socket.gethostname(), self.os.getpid(), - self.random.randint(0, 99999) + extra = "" + if _TEST_NAME_FILE: + f = open(_TEST_NAME_FILE) + test_name = f.read() + f.close() + extra = "." + test_name + data_suffix = "%s%s.%s.%06d" % ( + socket.gethostname(), extra, os.getpid(), + random.randint(0, 999999) ) self._harvest_data() @@ -465,7 +503,14 @@ class coverage(object): current measurements. """ - self.data.combine_parallel_data() + aliases = None + if self.config.paths: + aliases = PathAliases(self.file_locator) + for paths in self.config.paths.values(): + result = paths[0] + for pattern in paths[1:]: + aliases.add(pattern, result) + self.data.combine_parallel_data(aliases=aliases) def _harvest_data(self): """Get the collected data and reset the collector. @@ -473,27 +518,37 @@ class coverage(object): Also warn about various problems collecting data. """ - if not self._harvested: - self.data.add_line_data(self.collector.get_line_data()) - self.data.add_arc_data(self.collector.get_arc_data()) - self.collector.reset() + if not self._measured: + return - # If there are still entries in the source_pkgs list, then we never - # encountered those packages. + self.data.add_line_data(self.collector.get_line_data()) + self.data.add_arc_data(self.collector.get_arc_data()) + self.collector.reset() + + # If there are still entries in the source_pkgs list, then we never + # encountered those packages. + if self._warn_unimported_source: for pkg in self.source_pkgs: self._warn("Module %s was never imported." % pkg) - # Find out if we got any data. - summary = self.data.summary() - if not summary: - self._warn("No data was collected.") + # Find out if we got any data. + summary = self.data.summary() + if not summary and self._warn_no_data: + self._warn("No data was collected.") - # Find files that were never executed at all. - for src in self.source: - for py_file in find_python_files(src): - self.data.touch_file(py_file) + # Find files that were never executed at all. + for src in self.source: + for py_file in find_python_files(src): + py_file = self.file_locator.canonical_filename(py_file) - self._harvested = True + if self.omit_match and self.omit_match.match(py_file): + # Turns out this file was omitted, so don't pull it back + # in as unexecuted. + continue + + self.data.touch_file(py_file) + + self._measured = False # Backward compatibility with version 1. def analysis(self, morf): @@ -520,8 +575,11 @@ class coverage(object): """ analysis = self._analyze(morf) return ( - analysis.filename, analysis.statements, analysis.excluded, - analysis.missing, analysis.missing_formatted() + analysis.filename, + sorted(analysis.statements), + sorted(analysis.excluded), + sorted(analysis.missing), + analysis.missing_formatted(), ) def _analyze(self, it): @@ -530,6 +588,7 @@ class coverage(object): Returns an `Analysis` object. """ + self._harvest_data() if not isinstance(it, CodeUnit): it = code_unit_factory(it, self.file_locator)[0] @@ -548,14 +607,16 @@ class coverage(object): match those patterns will be included in the report. Modules matching `omit` will not be included in the report. + Returns a float, the total percentage covered. + """ + self._harvest_data() self.config.from_args( - ignore_errors=ignore_errors, omit=omit, include=include + ignore_errors=ignore_errors, omit=omit, include=include, + show_missing=show_missing, ) - reporter = SummaryReporter( - self, show_missing, self.config.ignore_errors - ) - reporter.report(morfs, outfile=file, config=self.config) + reporter = SummaryReporter(self, self.config) + return reporter.report(morfs, outfile=file) def annotate(self, morfs=None, directory=None, ignore_errors=None, omit=None, include=None): @@ -569,25 +630,39 @@ class coverage(object): See `coverage.report()` for other arguments. """ + self._harvest_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include ) - reporter = AnnotateReporter(self, self.config.ignore_errors) - reporter.report(morfs, config=self.config, directory=directory) + reporter = AnnotateReporter(self, self.config) + reporter.report(morfs, directory=directory) def html_report(self, morfs=None, directory=None, ignore_errors=None, - omit=None, include=None): + omit=None, include=None, extra_css=None, title=None): """Generate an HTML report. + The HTML is written to `directory`. The file "index.html" is the + overview starting point, with links to more detailed pages for + individual modules. + + `extra_css` is a path to a file of other CSS to apply on the page. + It will be copied into the HTML directory. + + `title` is a text string (not HTML) to use as the title of the HTML + report. + See `coverage.report()` for other arguments. + Returns a float, the total percentage covered. + """ + self._harvest_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include, - html_dir=directory, + html_dir=directory, extra_css=extra_css, html_title=title, ) - reporter = HtmlReporter(self, self.config.ignore_errors) - reporter.report(morfs, config=self.config) + reporter = HtmlReporter(self, self.config) + return reporter.report(morfs) def xml_report(self, morfs=None, outfile=None, ignore_errors=None, omit=None, include=None): @@ -600,12 +675,16 @@ class coverage(object): See `coverage.report()` for other arguments. + Returns a float, the total percentage covered. + """ + self._harvest_data() self.config.from_args( ignore_errors=ignore_errors, omit=omit, include=include, xml_output=outfile, ) file_to_close = None + delete_file = False if self.config.xml_output: if self.config.xml_output == '-': outfile = sys.stdout @@ -613,11 +692,17 @@ class coverage(object): outfile = open(self.config.xml_output, "w") file_to_close = outfile try: - reporter = XmlReporter(self, self.config.ignore_errors) - reporter.report(morfs, outfile=outfile, config=self.config) + try: + reporter = XmlReporter(self, self.config) + return reporter.report(morfs, outfile=outfile) + except CoverageException: + delete_file = True + raise finally: if file_to_close: file_to_close.close() + if delete_file: + file_be_gone(self.config.xml_output) def sysinfo(self): """Return a list of (key, value) pairs showing internal information.""" @@ -625,22 +710,43 @@ class coverage(object): import coverage as covmod import platform, re + try: + implementation = platform.python_implementation() + except AttributeError: + implementation = "unknown" + info = [ ('version', covmod.__version__), ('coverage', covmod.__file__), ('cover_dir', self.cover_dir), ('pylib_dirs', self.pylib_dirs), ('tracer', self.collector.tracer_name()), + ('config_files', self.config.attempted_config_files), + ('configs_read', self.config.config_files), ('data_path', self.data.filename), ('python', sys.version.replace('\n', '')), ('platform', platform.platform()), + ('implementation', implementation), + ('executable', sys.executable), ('cwd', os.getcwd()), ('path', sys.path), - ('environment', [ - ("%s = %s" % (k, v)) for k, v in os.environ.items() - if re.search("^COV|^PY", k) - ]), + ('environment', sorted([ + ("%s = %s" % (k, v)) for k, v in iitems(os.environ) + if re.search(r"^COV|^PY", k) + ])), + ('command_line', " ".join(getattr(sys, 'argv', ['???']))), ] + if self.source_match: + info.append(('source_match', self.source_match.info())) + if self.include_match: + info.append(('include_match', self.include_match.info())) + if self.omit_match: + info.append(('omit_match', self.omit_match.info())) + if self.cover_match: + info.append(('cover_match', self.cover_match.info())) + if self.pylib_match: + info.append(('pylib_match', self.pylib_match.info())) + return info @@ -667,7 +773,10 @@ def process_startup(): cps = os.environ.get("COVERAGE_PROCESS_START") if cps: cov = coverage(config_file=cps, auto_data=True) - if os.environ.get("COVERAGE_COVERAGE"): - # Measuring coverage within coverage.py takes yet more trickery. - cov.cover_dir = "Please measure coverage.py!" cov.start() + cov._warn_no_data = False + cov._warn_unimported_source = False + + +# A hack for debugging testing in subprocesses. +_TEST_NAME_FILE = "" #"/tmp/covtest.txt" diff --git a/python/helpers/coverage/data.py b/python/helpers/coverage/data.py index 3263cb38883b..fb88c5b1e638 100644 --- a/python/helpers/coverage/data.py +++ b/python/helpers/coverage/data.py @@ -2,7 +2,9 @@ import os -from coverage.backward import pickle, sorted # pylint: disable=W0622 +from coverage.backward import iitems, pickle, sorted # pylint: disable=W0622 +from coverage.files import PathAliases +from coverage.misc import file_be_gone class CoverageData(object): @@ -21,15 +23,18 @@ class CoverageData(object): """ - def __init__(self, basename=None, collector=None): + def __init__(self, basename=None, collector=None, debug=None): """Create a CoverageData. `basename` is the name of the file to use for storing data. `collector` is a string describing the coverage measurement software. + `debug` is a `DebugControl` object for writing debug messages. + """ self.collector = collector or 'unknown' + self.debug = debug self.use_file = True @@ -59,10 +64,6 @@ class CoverageData(object): # self.arcs = {} - self.os = os - self.sorted = sorted - self.pickle = pickle - def usefile(self, use_file=True): """Set whether or not to use a disk file for data.""" self.use_file = use_file @@ -92,21 +93,21 @@ class CoverageData(object): def erase(self): """Erase the data, both in this object, and from its file storage.""" if self.use_file: - if self.filename and os.path.exists(self.filename): - os.remove(self.filename) + if self.filename: + file_be_gone(self.filename) self.lines = {} self.arcs = {} def line_data(self): """Return the map from filenames to lists of line numbers executed.""" return dict( - [(f, self.sorted(lmap.keys())) for f, lmap in self.lines.items()] + [(f, sorted(lmap.keys())) for f, lmap in iitems(self.lines)] ) def arc_data(self): """Return the map from filenames to lists of line number pairs.""" return dict( - [(f, self.sorted(amap.keys())) for f, amap in self.arcs.items()] + [(f, sorted(amap.keys())) for f, amap in iitems(self.arcs)] ) def write_file(self, filename): @@ -123,10 +124,13 @@ class CoverageData(object): if self.collector: data['collector'] = self.collector + if self.debug and self.debug.should('dataio'): + self.debug.write("Writing data to %r" % (filename,)) + # Write the pickle to the file. fdata = open(filename, 'wb') try: - self.pickle.dump(data, fdata, 2) + pickle.dump(data, fdata, 2) finally: fdata.close() @@ -136,6 +140,8 @@ class CoverageData(object): def raw_data(self, filename): """Return the raw pickled data from `filename`.""" + if self.debug and self.debug.should('dataio'): + self.debug.write("Reading data from %r" % (filename,)) fdata = open(filename, 'rb') try: data = pickle.load(fdata) @@ -158,33 +164,39 @@ class CoverageData(object): # Unpack the 'lines' item. lines = dict([ (f, dict.fromkeys(linenos, None)) - for f, linenos in data.get('lines', {}).items() + for f, linenos in iitems(data.get('lines', {})) ]) # Unpack the 'arcs' item. arcs = dict([ (f, dict.fromkeys(arcpairs, None)) - for f, arcpairs in data.get('arcs', {}).items() + for f, arcpairs in iitems(data.get('arcs', {})) ]) except Exception: pass return lines, arcs - def combine_parallel_data(self): + def combine_parallel_data(self, aliases=None): """Combine a number of data files together. Treat `self.filename` as a file prefix, and combine the data from all of the data files starting with that prefix plus a dot. + If `aliases` is provided, it's a `PathAliases` object that is used to + re-map paths to match the local machine's. + """ + aliases = aliases or PathAliases() data_dir, local = os.path.split(self.filename) localdot = local + '.' for f in os.listdir(data_dir or '.'): if f.startswith(localdot): full_path = os.path.join(data_dir, f) new_lines, new_arcs = self._read_file(full_path) - for filename, file_data in new_lines.items(): + for filename, file_data in iitems(new_lines): + filename = aliases.map(filename) self.lines.setdefault(filename, {}).update(file_data) - for filename, file_data in new_arcs.items(): + for filename, file_data in iitems(new_arcs): + filename = aliases.map(filename) self.arcs.setdefault(filename, {}).update(file_data) if f != local: os.remove(full_path) @@ -195,7 +207,7 @@ class CoverageData(object): `line_data` is { filename: { lineno: None, ... }, ...} """ - for filename, linenos in line_data.items(): + for filename, linenos in iitems(line_data): self.lines.setdefault(filename, {}).update(linenos) def add_arc_data(self, arc_data): @@ -204,7 +216,7 @@ class CoverageData(object): `arc_data` is { filename: { (l1,l2): None, ... }, ...} """ - for filename, arcs in arc_data.items(): + for filename, arcs in iitems(arc_data): self.arcs.setdefault(filename, {}).update(arcs) def touch_file(self, filename): @@ -245,8 +257,8 @@ class CoverageData(object): if fullpath: filename_fn = lambda f: f else: - filename_fn = self.os.path.basename - for filename, lines in self.lines.items(): + filename_fn = os.path.basename + for filename, lines in iitems(self.lines): summ[filename_fn(filename)] = len(lines) return summ diff --git a/python/helpers/coverage/debug.py b/python/helpers/coverage/debug.py new file mode 100644 index 000000000000..104f3b1d0a43 --- /dev/null +++ b/python/helpers/coverage/debug.py @@ -0,0 +1,54 @@ +"""Control of and utilities for debugging.""" + +import os + + +# When debugging, it can be helpful to force some options, especially when +# debugging the configuration mechanisms you usually use to control debugging! +# This is a list of forced debugging options. +FORCED_DEBUG = [] + + +class DebugControl(object): + """Control and output for debugging.""" + + def __init__(self, options, output): + """Configure the options and output file for debugging.""" + self.options = options + self.output = output + + def should(self, option): + """Decide whether to output debug information in category `option`.""" + return (option in self.options or option in FORCED_DEBUG) + + def write(self, msg): + """Write a line of debug output.""" + if self.should('pid'): + msg = "pid %5d: %s" % (os.getpid(), msg) + self.output.write(msg+"\n") + self.output.flush() + + def write_formatted_info(self, info): + """Write a sequence of (label,data) pairs nicely.""" + for line in info_formatter(info): + self.write(" %s" % line) + + +def info_formatter(info): + """Produce a sequence of formatted lines from info. + + `info` is a sequence of pairs (label, data). The produced lines are + nicely formatted, ready to print. + + """ + label_len = max([len(l) for l, _d in info]) + for label, data in info: + if data == []: + data = "-none-" + if isinstance(data, (list, tuple)): + prefix = "%*s:" % (label_len, label) + for e in data: + yield "%*s %s" % (label_len+1, prefix, e) + prefix = "" + else: + yield "%*s: %s" % (label_len, label, data) diff --git a/python/helpers/coverage/execfile.py b/python/helpers/coverage/execfile.py index 71227b715a3e..569204eac7d5 100644 --- a/python/helpers/coverage/execfile.py +++ b/python/helpers/coverage/execfile.py @@ -1,9 +1,11 @@ """Execute files of Python code.""" -import imp, os, sys - +import imp +import marshal +import os +import sys from coverage.backward import exec_code_object, open_source -from coverage.misc import NoSource, ExceptionDuringRun +from coverage.misc import ExceptionDuringRun, NoCode, NoSource try: @@ -65,6 +67,8 @@ def run_python_module(modulename, args): openfile.close() # Finally, hand the file off to run_python_file for execution. + pathname = os.path.abspath(pathname) + args[0] = pathname run_python_file(pathname, args, package=packagename) @@ -82,34 +86,22 @@ def run_python_file(filename, args, package=None): main_mod = imp.new_module('__main__') sys.modules['__main__'] = main_mod main_mod.__file__ = filename - main_mod.__package__ = package + if package: + main_mod.__package__ = package main_mod.__builtins__ = BUILTINS - # Set sys.argv and the first path element properly. + # Set sys.argv properly. old_argv = sys.argv - old_path0 = sys.path[0] sys.argv = args - sys.path[0] = os.path.abspath(os.path.dirname(filename)) try: - # Open the source file. - try: - source_file = open_source(filename) - except IOError: - raise NoSource("No file to run: %r" % filename) + # Make a code object somehow. + if filename.endswith(".pyc") or filename.endswith(".pyo"): + code = make_code_from_pyc(filename) + else: + code = make_code_from_py(filename) - try: - source = source_file.read() - finally: - source_file.close() - - # We have the source. `compile` still needs the last line to be clean, - # so make sure it is, then compile a code object from it. - if source[-1] != '\n': - source += '\n' - code = compile(source, filename, "exec") - - # Execute the source file. + # Execute the code object. try: exec_code_object(code, main_mod.__dict__) except SystemExit: @@ -130,4 +122,52 @@ def run_python_file(filename, args, package=None): # Restore the old argv and path sys.argv = old_argv - sys.path[0] = old_path0 + +def make_code_from_py(filename): + """Get source from `filename` and make a code object of it.""" + # Open the source file. + try: + source_file = open_source(filename) + except IOError: + raise NoSource("No file to run: %r" % filename) + + try: + source = source_file.read() + finally: + source_file.close() + + # We have the source. `compile` still needs the last line to be clean, + # so make sure it is, then compile a code object from it. + if not source or source[-1] != '\n': + source += '\n' + code = compile(source, filename, "exec") + + return code + + +def make_code_from_pyc(filename): + """Get a code object from a .pyc file.""" + try: + fpyc = open(filename, "rb") + except IOError: + raise NoCode("No file to run: %r" % filename) + + try: + # First four bytes are a version-specific magic number. It has to + # match or we won't run the file. + magic = fpyc.read(4) + if magic != imp.get_magic(): + raise NoCode("Bad magic number in .pyc file") + + # Skip the junk in the header that we don't need. + fpyc.read(4) # Skip the moddate. + if sys.version_info >= (3, 3): + # 3.3 added another long to the header (size), skip it. + fpyc.read(4) + + # The rest of the file is the code object we want. + code = marshal.load(fpyc) + finally: + fpyc.close() + + return code diff --git a/python/helpers/coverage/files.py b/python/helpers/coverage/files.py index a68a0a7fef88..addcf6768129 100644 --- a/python/helpers/coverage/files.py +++ b/python/helpers/coverage/files.py @@ -1,23 +1,27 @@ """File wrangling.""" from coverage.backward import to_string -import fnmatch, os, sys +from coverage.misc import CoverageException +import fnmatch +import os +import os.path +import re +import sys +import ntpath +import posixpath + class FileLocator(object): """Understand how filenames work.""" def __init__(self): # The absolute path to our current directory. - self.relative_dir = self.abs_file(os.curdir) + os.sep + self.relative_dir = os.path.normcase(abs_file(os.curdir) + os.sep) # Cache of results of calling the canonical_filename() method, to # avoid duplicating work. self.canonical_filename_cache = {} - def abs_file(self, filename): - """Return the absolute normalized form of `filename`.""" - return os.path.normcase(os.path.abspath(os.path.realpath(filename))) - def relative_filename(self, filename): """Return the relative form of `filename`. @@ -25,8 +29,9 @@ class FileLocator(object): `FileLocator` was constructed. """ - if filename.startswith(self.relative_dir): - filename = filename.replace(self.relative_dir, "") + fnorm = os.path.normcase(filename) + if fnorm.startswith(self.relative_dir): + filename = filename[len(self.relative_dir):] return filename def canonical_filename(self, filename): @@ -36,19 +41,15 @@ class FileLocator(object): """ if filename not in self.canonical_filename_cache: - f = filename - if os.path.isabs(f) and not os.path.exists(f): - if self.get_zip_data(f) is None: - f = os.path.basename(f) - if not os.path.isabs(f): + if not os.path.isabs(filename): for path in [os.curdir] + sys.path: if path is None: continue - g = os.path.join(path, f) - if os.path.exists(g): - f = g + f = os.path.join(path, filename) + if os.path.exists(f): + filename = f break - cf = self.abs_file(f) + cf = abs_file(filename) self.canonical_filename_cache[filename] = cf return self.canonical_filename_cache[filename] @@ -77,6 +78,78 @@ class FileLocator(object): return None +if sys.platform == 'win32': + + def actual_path(path): + """Get the actual path of `path`, including the correct case.""" + if path in actual_path.cache: + return actual_path.cache[path] + + head, tail = os.path.split(path) + if not tail: + actpath = head + elif not head: + actpath = tail + else: + head = actual_path(head) + if head in actual_path.list_cache: + files = actual_path.list_cache[head] + else: + try: + files = os.listdir(head) + except OSError: + files = [] + actual_path.list_cache[head] = files + normtail = os.path.normcase(tail) + for f in files: + if os.path.normcase(f) == normtail: + tail = f + break + actpath = os.path.join(head, tail) + actual_path.cache[path] = actpath + return actpath + + actual_path.cache = {} + actual_path.list_cache = {} + +else: + def actual_path(filename): + """The actual path for non-Windows platforms.""" + return filename + + +def abs_file(filename): + """Return the absolute normalized form of `filename`.""" + path = os.path.expandvars(os.path.expanduser(filename)) + path = os.path.abspath(os.path.realpath(path)) + path = actual_path(path) + return path + + +def isabs_anywhere(filename): + """Is `filename` an absolute path on any OS?""" + return ntpath.isabs(filename) or posixpath.isabs(filename) + + +def prep_patterns(patterns): + """Prepare the file patterns for use in a `FnmatchMatcher`. + + If a pattern starts with a wildcard, it is used as a pattern + as-is. If it does not start with a wildcard, then it is made + absolute with the current directory. + + If `patterns` is None, an empty list is returned. + + """ + prepped = [] + for p in patterns or []: + if p.startswith("*") or p.startswith("?"): + prepped.append(p) + else: + prepped.append(abs_file(p)) + return prepped + + class TreeMatcher(object): """A matcher for files in a tree.""" def __init__(self, directories): @@ -85,6 +158,10 @@ class TreeMatcher(object): def __repr__(self): return "" % self.dirs + def info(self): + """A list of strings for displaying when dumping state.""" + return self.dirs + def add(self, directory): """Add another directory to the list we match for.""" self.dirs.append(directory) @@ -110,6 +187,10 @@ class FnmatchMatcher(object): def __repr__(self): return "" % self.pats + def info(self): + """A list of strings for displaying when dumping state.""" + return self.pats + def match(self, fpath): """Does `fpath` match one of our filename patterns?""" for pat in self.pats: @@ -118,14 +199,117 @@ class FnmatchMatcher(object): return False +def sep(s): + """Find the path separator used in this string, or os.sep if none.""" + sep_match = re.search(r"[\\/]", s) + if sep_match: + the_sep = sep_match.group(0) + else: + the_sep = os.sep + return the_sep + + +class PathAliases(object): + """A collection of aliases for paths. + + When combining data files from remote machines, often the paths to source + code are different, for example, due to OS differences, or because of + serialized checkouts on continuous integration machines. + + A `PathAliases` object tracks a list of pattern/result pairs, and can + map a path through those aliases to produce a unified path. + + `locator` is a FileLocator that is used to canonicalize the results. + + """ + def __init__(self, locator=None): + self.aliases = [] + self.locator = locator + + def add(self, pattern, result): + """Add the `pattern`/`result` pair to the list of aliases. + + `pattern` is an `fnmatch`-style pattern. `result` is a simple + string. When mapping paths, if a path starts with a match against + `pattern`, then that match is replaced with `result`. This models + isomorphic source trees being rooted at different places on two + different machines. + + `pattern` can't end with a wildcard component, since that would + match an entire tree, and not just its root. + + """ + # The pattern can't end with a wildcard component. + pattern = pattern.rstrip(r"\/") + if pattern.endswith("*"): + raise CoverageException("Pattern must not end with wildcards.") + pattern_sep = sep(pattern) + + # The pattern is meant to match a filepath. Let's make it absolute + # unless it already is, or is meant to match any prefix. + if not pattern.startswith('*') and not isabs_anywhere(pattern): + pattern = abs_file(pattern) + pattern += pattern_sep + + # Make a regex from the pattern. fnmatch always adds a \Z or $ to + # match the whole string, which we don't want. + regex_pat = fnmatch.translate(pattern).replace(r'\Z(', '(') + if regex_pat.endswith("$"): + regex_pat = regex_pat[:-1] + # We want */a/b.py to match on Windows too, so change slash to match + # either separator. + regex_pat = regex_pat.replace(r"\/", r"[\\/]") + # We want case-insensitive matching, so add that flag. + regex = re.compile(r"(?i)" + regex_pat) + + # Normalize the result: it must end with a path separator. + result_sep = sep(result) + result = result.rstrip(r"\/") + result_sep + self.aliases.append((regex, result, pattern_sep, result_sep)) + + def map(self, path): + """Map `path` through the aliases. + + `path` is checked against all of the patterns. The first pattern to + match is used to replace the root of the path with the result root. + Only one pattern is ever used. If no patterns match, `path` is + returned unchanged. + + The separator style in the result is made to match that of the result + in the alias. + + """ + for regex, result, pattern_sep, result_sep in self.aliases: + m = regex.match(path) + if m: + new = path.replace(m.group(0), result) + if pattern_sep != result_sep: + new = new.replace(pattern_sep, result_sep) + if self.locator: + new = self.locator.canonical_filename(new) + return new + return path + + def find_python_files(dirname): - """Yield all of the importable Python files in `dirname`, recursively.""" - for dirpath, dirnames, filenames in os.walk(dirname, topdown=True): - if '__init__.py' not in filenames: + """Yield all of the importable Python files in `dirname`, recursively. + + To be importable, the files have to be in a directory with a __init__.py, + except for `dirname` itself, which isn't required to have one. The + assumption is that `dirname` was specified directly, so the user knows + best, but subdirectories are checked for a __init__.py to be sure we only + find the importable files. + + """ + for i, (dirpath, dirnames, filenames) in enumerate(os.walk(dirname)): + if i > 0 and '__init__.py' not in filenames: # If a directory doesn't have __init__.py, then it isn't # importable and neither are its files del dirnames[:] continue for filename in filenames: - if fnmatch.fnmatch(filename, "*.py"): + # We're only interested in files that look like reasonable Python + # files: Must end with .py or .pyw, and must not have certain funny + # characters that probably mean they are editor junk. + if re.match(r"^[^.#~!$@%^&*()+=,]+\.pyw?$", filename): yield os.path.join(dirpath, filename) diff --git a/python/helpers/coverage/fullcoverage/encodings.py b/python/helpers/coverage/fullcoverage/encodings.py new file mode 100644 index 000000000000..b7e0435af87c --- /dev/null +++ b/python/helpers/coverage/fullcoverage/encodings.py @@ -0,0 +1,56 @@ +"""Imposter encodings module that installs a coverage-style tracer. + +This is NOT the encodings module; it is an imposter that sets up tracing +instrumentation and then replaces itself with the real encodings module. + +If the directory that holds this file is placed first in the PYTHONPATH when +using "coverage" to run Python's tests, then this file will become the very +first module imported by the internals of Python 3. It installs a +coverage-compatible trace function that can watch Standard Library modules +execute from the very earliest stages of Python's own boot process. This fixes +a problem with coverage - that it starts too late to trace the coverage of many +of the most fundamental modules in the Standard Library. + +""" + +import sys + +class FullCoverageTracer(object): + def __init__(self): + # `traces` is a list of trace events. Frames are tricky: the same + # frame object is used for a whole scope, with new line numbers + # written into it. So in one scope, all the frame objects are the + # same object, and will eventually all will point to the last line + # executed. So we keep the line numbers alongside the frames. + # The list looks like: + # + # traces = [ + # ((frame, event, arg), lineno), ... + # ] + # + self.traces = [] + + def fullcoverage_trace(self, *args): + frame, event, arg = args + self.traces.append((args, frame.f_lineno)) + return self.fullcoverage_trace + +sys.settrace(FullCoverageTracer().fullcoverage_trace) + +# In coverage/files.py is actual_filename(), which uses glob.glob. I don't +# understand why, but that use of glob borks everything if fullcoverage is in +# effect. So here we make an ugly hail-mary pass to switch off glob.glob over +# there. This means when using fullcoverage, Windows path names will not be +# their actual case. + +#sys.fullcoverage = True + +# Finally, remove our own directory from sys.path; remove ourselves from +# sys.modules; and re-import "encodings", which will be the real package +# this time. Note that the delete from sys.modules dictionary has to +# happen last, since all of the symbols in this module will become None +# at that exact moment, including "sys". + +parentdir = max(filter(__file__.startswith, sys.path), key=len) +sys.path.remove(parentdir) +del sys.modules['encodings'] diff --git a/python/helpers/coverage/html.py b/python/helpers/coverage/html.py index fffd9b45b41a..b58bfb7ee3be 100644 --- a/python/helpers/coverage/html.py +++ b/python/helpers/coverage/html.py @@ -1,21 +1,47 @@ """HTML reporting for Coverage.""" -import os, re, shutil - +import os +import re +import shutil +import sys import coverage from coverage.backward import pickle from coverage.misc import CoverageException, Hasher -from coverage.phystokens import source_token_lines +from coverage.phystokens import source_token_lines, source_encoding from coverage.report import Reporter +from coverage.results import Numbers from coverage.templite import Templite -# Disable pylint msg W0612, because a bunch of variables look unused, but -# they're accessed in a Templite context via locals(). -# pylint: disable=W0612 -def data_filename(fname): - """Return the path to a data file of ours.""" - return os.path.join(os.path.split(__file__)[0], fname) +# Static files are looked for in a list of places. +STATIC_PATH = [ + # The place Debian puts system Javascript libraries. + "/usr/share/javascript", + + # Our htmlfiles directory. + os.path.join(os.path.dirname(__file__), "htmlfiles"), +] + +def data_filename(fname, pkgdir=""): + """Return the path to a data file of ours. + + The file is searched for on `STATIC_PATH`, and the first place it's found, + is returned. + + Each directory in `STATIC_PATH` is searched as-is, and also, if `pkgdir` + is provided, at that subdirectory. + + """ + for static_dir in STATIC_PATH: + static_filename = os.path.join(static_dir, fname) + if os.path.exists(static_filename): + return static_filename + if pkgdir: + static_filename = os.path.join(static_dir, pkgdir, fname) + if os.path.exists(static_filename): + return static_filename + raise CoverageException("Couldn't find static file %r" % fname) + def data(fname): """Return the contents of a data file of ours.""" @@ -31,26 +57,27 @@ class HtmlReporter(Reporter): # These files will be copied from the htmlfiles dir to the output dir. STATIC_FILES = [ - "style.css", - "jquery-1.4.3.min.js", - "jquery.hotkeys.js", - "jquery.isonscreen.js", - "jquery.tablesorter.min.js", - "coverage_html.js", - "keybd_closed.png", - "keybd_open.png", + ("style.css", ""), + ("jquery.min.js", "jquery"), + ("jquery.hotkeys.js", "jquery-hotkeys"), + ("jquery.isonscreen.js", "jquery-isonscreen"), + ("jquery.tablesorter.min.js", "jquery-tablesorter"), + ("coverage_html.js", ""), + ("keybd_closed.png", ""), + ("keybd_open.png", ""), ] - def __init__(self, cov, ignore_errors=False): - super(HtmlReporter, self).__init__(cov, ignore_errors) + def __init__(self, cov, config): + super(HtmlReporter, self).__init__(cov, config) self.directory = None self.template_globals = { 'escape': escape, + 'title': self.config.html_title, '__url__': coverage.__url__, '__version__': coverage.__version__, } self.source_tmpl = Templite( - data("htmlfiles/pyfile.html"), self.template_globals + data("pyfile.html"), self.template_globals ) self.coverage = cov @@ -58,29 +85,34 @@ class HtmlReporter(Reporter): self.files = [] self.arcs = self.coverage.data.has_arcs() self.status = HtmlStatus() + self.extra_css = None + self.totals = Numbers() - def report(self, morfs, config=None): + def report(self, morfs): """Generate an HTML report for `morfs`. - `morfs` is a list of modules or filenames. `config` is a - CoverageConfig instance. + `morfs` is a list of modules or filenames. """ - assert config.html_dir, "must provide a directory for html reporting" + assert self.config.html_dir, "must give a directory for html reporting" # Read the status data. - self.status.read(config.html_dir) + self.status.read(self.config.html_dir) # Check that this run used the same settings as the last run. m = Hasher() - m.update(config) + m.update(self.config) these_settings = m.digest() if self.status.settings_hash() != these_settings: self.status.reset() self.status.set_settings_hash(these_settings) + # The user may have extra CSS they want copied. + if self.config.extra_css: + self.extra_css = os.path.basename(self.config.extra_css) + # Process all the files. - self.report_files(self.html_file, morfs, config, config.html_dir) + self.report_files(self.html_file, morfs, self.config.html_dir) if not self.files: raise CoverageException("No data to report.") @@ -88,13 +120,34 @@ class HtmlReporter(Reporter): # Write the index file. self.index_file() - # Create the once-per-directory files. - for static in self.STATIC_FILES: + self.make_local_static_report_files() + + return self.totals.pc_covered + + def make_local_static_report_files(self): + """Make local instances of static files for HTML report.""" + # The files we provide must always be copied. + for static, pkgdir in self.STATIC_FILES: shutil.copyfile( - data_filename("htmlfiles/" + static), + data_filename(static, pkgdir), os.path.join(self.directory, static) ) + # The user may have extra CSS they want copied. + if self.extra_css: + shutil.copyfile( + self.config.extra_css, + os.path.join(self.directory, self.extra_css) + ) + + def write_html(self, fname, html): + """Write `html` to `fname`, properly encoded.""" + fout = open(fname, "wb") + try: + fout.write(html.encode('ascii', 'xmlcharrefreplace')) + finally: + fout.close() + def file_hash(self, source, cu): """Compute a hash that changes if the file needs to be re-reported.""" m = Hasher() @@ -121,11 +174,20 @@ class HtmlReporter(Reporter): self.status.set_file_hash(flat_rootname, this_hash) + # If need be, determine the encoding of the source file. We use it + # later to properly write the HTML. + if sys.version_info < (3, 0): + encoding = source_encoding(source) + # Some UTF8 files have the dreaded UTF8 BOM. If so, junk it. + if encoding.startswith("utf-8") and source[:3] == "\xef\xbb\xbf": + source = source[3:] + encoding = "utf-8" + + # Get the numbers for this file. nums = analysis.numbers - missing_branch_arcs = analysis.missing_branch_arcs() - n_par = 0 # accumulated below. - arcs = self.arcs + if self.arcs: + missing_branch_arcs = analysis.missing_branch_arcs() # These classes determine which lines are highlighted by default. c_run = "run hide_run" @@ -149,7 +211,6 @@ class HtmlReporter(Reporter): line_class.append(c_mis) elif self.arcs and lineno in missing_branch_arcs: line_class.append(c_par) - n_par += 1 annlines = [] for b in missing_branch_arcs[lineno]: if b < 0: @@ -184,19 +245,22 @@ class HtmlReporter(Reporter): }) # Write the HTML page for this file. + html = spaceless(self.source_tmpl.render({ + 'c_exc': c_exc, 'c_mis': c_mis, 'c_par': c_par, 'c_run': c_run, + 'arcs': self.arcs, 'extra_css': self.extra_css, + 'cu': cu, 'nums': nums, 'lines': lines, + })) + + if sys.version_info < (3, 0): + html = html.decode(encoding) + html_filename = flat_rootname + ".html" html_path = os.path.join(self.directory, html_filename) - html = spaceless(self.source_tmpl.render(locals())) - fhtml = open(html_path, 'w') - try: - fhtml.write(html) - finally: - fhtml.close() + self.write_html(html_path, html) # Save this file's information for the index file. index_info = { 'nums': nums, - 'par': n_par, 'html_filename': html_filename, 'name': cu.name, } @@ -206,19 +270,24 @@ class HtmlReporter(Reporter): def index_file(self): """Write the index.html file for this report.""" index_tmpl = Templite( - data("htmlfiles/index.html"), self.template_globals + data("index.html"), self.template_globals ) - files = self.files - arcs = self.arcs + self.totals = sum([f['nums'] for f in self.files]) - totals = sum([f['nums'] for f in files]) + html = index_tmpl.render({ + 'arcs': self.arcs, + 'extra_css': self.extra_css, + 'files': self.files, + 'totals': self.totals, + }) - fhtml = open(os.path.join(self.directory, "index.html"), "w") - try: - fhtml.write(index_tmpl.render(locals())) - finally: - fhtml.close() + if sys.version_info < (3, 0): + html = html.decode("utf-8") + self.write_html( + os.path.join(self.directory, "index.html"), + html + ) # Write the latest hashes for next time. self.status.write(self.directory) @@ -243,8 +312,12 @@ class HtmlStatus(object): usable = False try: status_file = os.path.join(directory, self.STATUS_FILE) - status = pickle.load(open(status_file, "rb")) - except IOError: + fstatus = open(status_file, "rb") + try: + status = pickle.load(fstatus) + finally: + fstatus.close() + except (IOError, ValueError): usable = False else: usable = True @@ -321,5 +394,5 @@ def spaceless(html): Get rid of some. """ - html = re.sub(">\s+

\n

\s+

\n

- Coverage report + {{ title|escape }} - + {% if extra_css %} + + {% endif %} + @@ -16,7 +19,7 @@