Coverage.py updated to version 3.7.1

python/helpers/coverage/__init__.py

@@ -5,19 +5,13 @@ http://nedbatchelder.com/code/coverage
 
 """
 
-__version__ = "3.5"     # see detailed history in CHANGES.txt
-
-__url__ = "http://nedbatchelder.com/code/coverage"
-if max(__version__).isalpha():
-    # For pre-releases, use a version-specific URL.
-    __url__ += "/" + __version__
+from coverage.version import __version__, __url__
 
 from coverage.control import coverage, process_startup
 from coverage.data import CoverageData
 from coverage.cmdline import main, CoverageScript
 from coverage.misc import CoverageException
 
-
 # Module-level functions.  The original API to this module was based on
 # functions defined directly in the module, with a singleton of the coverage()
 # class.  That design hampered programmability, so the current api uses
@@ -36,12 +30,34 @@ def _singleton_method(name):
     called.
 
     """
+    # Disable pylint msg W0612, because a bunch of variables look unused, but
+    # they're accessed via locals().
+    # pylint: disable=W0612
+
     def wrapper(*args, **kwargs):
         """Singleton wrapper around a coverage method."""
         global _the_coverage
         if not _the_coverage:
             _the_coverage = coverage(auto_data=True)
         return getattr(_the_coverage, name)(*args, **kwargs)
+
+    import inspect
+    meth = getattr(coverage, name)
+    args, varargs, kw, defaults = inspect.getargspec(meth)
+    argspec = inspect.formatargspec(args[1:], varargs, kw, defaults)
+    docstring = meth.__doc__
+    wrapper.__doc__ = ("""\
+        A first-use-singleton wrapper around coverage.%(name)s.
+
+        This wrapper is provided for backward compatibility with legacy code.
+        New code should use coverage.%(name)s directly.
+
+        %(name)s%(argspec)s:
+
+        %(docstring)s
+        """ % locals()
+        )
+
     return wrapper
 
 
@@ -57,10 +73,25 @@ report =    _singleton_method('report')
 annotate =  _singleton_method('annotate')
 
 
+# On Windows, we encode and decode deep enough that something goes wrong and
+# the encodings.utf_8 module is loaded and then unloaded, I don't know why.
+# Adding a reference here prevents it from being unloaded.  Yuk.
+
+# Because of the "from coverage.control import fooey" lines at the top of the
+# file, there's an entry for coverage.coverage in sys.modules, mapped to None.
+# This makes some inspection tools (like pydoc) unable to find the class
+# coverage.coverage.  So remove that entry.
+import sys
+try:
+    del sys.modules['coverage.coverage']
+except KeyError:
+    pass
+
+
 # COPYRIGHT AND LICENSE
 #
 # Copyright 2001 Gareth Rees.  All rights reserved.
-# Copyright 2004-2010 Ned Batchelder.  All rights reserved.
+# Copyright 2004-2013 Ned Batchelder.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are

python/helpers/coverage/__main__.py

@@ -1,3 +1,4 @@
-"""Coverage.py's main entrypoint."""
+"""Coverage.py's main entry point."""
+import sys
 from coverage.cmdline import main
-main()
+sys.exit(main())

python/helpers/coverage/annotate.py

@@ -1,7 +1,8 @@
 """Source file annotation for Coverage."""
 
-import os, re
-
+import os
+import re
+from coverage.backward import sorted                    # pylint: disable=W0622
 from coverage.report import Reporter
 
 class AnnotateReporter(Reporter):
@@ -26,20 +27,20 @@ class AnnotateReporter(Reporter):
 
     """
 
-    def __init__(self, coverage, ignore_errors=False):
-        super(AnnotateReporter, self).__init__(coverage, ignore_errors)
+    def __init__(self, coverage, config):
+        super(AnnotateReporter, self).__init__(coverage, config)
         self.directory = None
 
     blank_re = re.compile(r"\s*(#|$)")
     else_re = re.compile(r"\s*else\s*:\s*(#|$)")
 
-    def report(self, morfs, config, directory=None):
+    def report(self, morfs, directory=None):
         """Run the report.
 
         See `coverage.report()` for arguments.
 
         """
-        self.report_files(self.annotate_file, morfs, config, directory)
+        self.report_files(self.annotate_file, morfs, directory)
 
     def annotate_file(self, cu, analysis):
         """Annotate a single file.
@@ -59,9 +60,9 @@ class AnnotateReporter(Reporter):
             dest_file = filename + ",cover"
         dest = open(dest_file, 'w')
 
-        statements = analysis.statements
-        missing = analysis.missing
-        excluded = analysis.excluded
+        statements = sorted(analysis.statements)
+        missing = sorted(analysis.missing)
+        excluded = sorted(analysis.excluded)
 
         lineno = 0
         i = 0

python/helpers/coverage/backward.py

@@ -6,8 +6,6 @@
 # W0611: Unused import blah
 # W0622: Redefining built-in blah
 
-import os, sys
-
 # Python 2.3 doesn't have `set`
 try:
     set = set       # new in 2.4
@@ -24,6 +22,31 @@ except NameError:
         lst.sort()
         return lst
 
+# Python 2.3 doesn't have `reversed`.
+try:
+    reversed = reversed
+except NameError:
+    def reversed(iterable):
+        """A 2.3-compatible implementation of `reversed`."""
+        lst = list(iterable)
+        return lst[::-1]
+
+# rpartition is new in 2.5
+try:
+    "".rpartition
+except AttributeError:
+    def rpartition(s, sep):
+        """Implement s.rpartition(sep) for old Pythons."""
+        i = s.rfind(sep)
+        if i == -1:
+            return ('', '', s)
+        else:
+            return (s[:i], sep, s[i+len(sep):])
+else:
+    def rpartition(s, sep):
+        """A common interface for new Pythons."""
+        return s.rpartition(sep)
+
 # Pythons 2 and 3 differ on where to get StringIO
 try:
     from cStringIO import StringIO
@@ -49,6 +72,18 @@ try:
 except NameError:
     range = range
 
+# A function to iterate listlessly over a dict's items.
+try:
+    {}.iteritems
+except AttributeError:
+    def iitems(d):
+        """Produce the items from dict `d`."""
+        return d.items()
+else:
+    def iitems(d):
+        """Produce the items from dict `d`."""
+        return d.iteritems()
+
 # Exec is a statement in Py2, a function in Py3
 if sys.version_info >= (3, 0):
     def exec_code_object(code, global_map):
@@ -66,21 +101,32 @@ else:
             )
         )
 
-# ConfigParser was renamed to the more-standard configparser
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
-
-# Python 3.2 provides `tokenize.open`, the best way to open source files.
-try:
+# Reading Python source and interpreting the coding comment is a big deal.
+if sys.version_info >= (3, 0):
+    # Python 3.2 provides `tokenize.open`, the best way to open source files.
     import tokenize
-    open_source = tokenize.open     # pylint: disable=E1101
-except AttributeError:
+    try:
+        open_source = tokenize.open     # pylint: disable=E1101
+    except AttributeError:
+        from io import TextIOWrapper
+        detect_encoding = tokenize.detect_encoding  # pylint: disable=E1101
+        # Copied from the 3.2 stdlib:
+        def open_source(fname):
+            """Open a file in read only mode using the encoding detected by
+            detect_encoding().
+            """
+            buffer = open(fname, 'rb')
+            encoding, _ = detect_encoding(buffer.readline)
+            buffer.seek(0)
+            text = TextIOWrapper(buffer, encoding, line_buffering=True)
+            text.mode = 'r'
+            return text
+else:
     def open_source(fname):
         """Open a source file the best way."""
         return open(fname, "rU")
 
+
 # Python 3.x is picky about bytes and strings, so provide methods to
 # get them right, and make them no-ops in 2.x
 if sys.version_info >= (3, 0):
@@ -92,6 +138,19 @@ if sys.version_info >= (3, 0):
         """Convert bytes `b` to a string."""
         return b.decode('utf8')
 
+    def binary_bytes(byte_values):
+        """Produce a byte string with the ints from `byte_values`."""
+        return bytes(byte_values)
+
+    def byte_to_int(byte_value):
+        """Turn an element of a bytes object into an int."""
+        return byte_value
+
+    def bytes_to_ints(bytes_value):
+        """Turn a bytes object into a sequence of ints."""
+        # In Py3, iterating bytes gives ints.
+        return bytes_value
+
 else:
     def to_bytes(s):
         """Convert string `s` to bytes (no-op in 2.x)."""
@@ -101,6 +160,19 @@ else:
         """Convert bytes `b` to a string (no-op in 2.x)."""
         return b
 
+    def binary_bytes(byte_values):
+        """Produce a byte string with the ints from `byte_values`."""
+        return "".join([chr(b) for b in byte_values])
+
+    def byte_to_int(byte_value):
+        """Turn an element of a bytes object into an int."""
+        return ord(byte_value)
+
+    def bytes_to_ints(bytes_value):
+        """Turn a bytes object into a sequence of ints."""
+        for byte in bytes_value:
+            yield ord(byte)
+
 # Md5 is available in different places.
 try:
     import hashlib

python/helpers/coverage/bytecode.py

@@ -1,14 +1,25 @@
 """Bytecode manipulation for coverage.py"""
 
-import opcode, sys, types
+import opcode
+import types
+from coverage.backward import byte_to_int
 
 class ByteCode(object):
     """A single bytecode."""
     def __init__(self):
+        # The offset of this bytecode in the code object.
         self.offset = -1
+
+        # The opcode, defined in the `opcode` module.
         self.op = -1
+
+        # The argument, a small integer, whose meaning depends on the opcode.
         self.arg = -1
+
+        # The offset in the code object of the next bytecode.
         self.next_offset = -1
+
+        # The offset to jump to.
         self.jump_to = -1
 
 
@@ -18,44 +29,34 @@ class ByteCodes(object):
     Returns `ByteCode` objects.
 
     """
+    # pylint: disable=R0924
     def __init__(self, code):
         self.code = code
-        self.offset = 0
 
-    if sys.version_info >= (3, 0):
-        def __getitem__(self, i):
-            return self.code[i]
-    else:
-        def __getitem__(self, i):
-            return ord(self.code[i])
+    def __getitem__(self, i):
+        return byte_to_int(self.code[i])
 
     def __iter__(self):
-        return self
+        offset = 0
+        while offset < len(self.code):
+            bc = ByteCode()
+            bc.op = self[offset]
+            bc.offset = offset
 
-    def __next__(self):
-        if self.offset >= len(self.code):
-            raise StopIteration
+            next_offset = offset+1
+            if bc.op >= opcode.HAVE_ARGUMENT:
+                bc.arg = self[offset+1] + 256*self[offset+2]
+                next_offset += 2
 
-        bc = ByteCode()
-        bc.op = self[self.offset]
-        bc.offset = self.offset
+                label = -1
+                if bc.op in opcode.hasjrel:
+                    label = next_offset + bc.arg
+                elif bc.op in opcode.hasjabs:
+                    label = bc.arg
+                bc.jump_to = label
 
-        next_offset = self.offset+1
-        if bc.op >= opcode.HAVE_ARGUMENT:
-            bc.arg = self[self.offset+1] + 256*self[self.offset+2]
-            next_offset += 2
-
-            label = -1
-            if bc.op in opcode.hasjrel:
-                label = next_offset + bc.arg
-            elif bc.op in opcode.hasjabs:
-                label = bc.arg
-            bc.jump_to = label
-
-        bc.next_offset = self.offset = next_offset
-        return bc
-
-    next = __next__     # Py2k uses an old-style non-dunder name.
+            bc.next_offset = offset = next_offset
+            yield bc
 
 
 class CodeObjects(object):
@@ -64,18 +65,11 @@ class CodeObjects(object):
         self.stack = [code]
 
     def __iter__(self):
-        return self
-
-    def __next__(self):
-        if self.stack:
+        while self.stack:
             # We're going to return the code object on the stack, but first
             # push its children for later returning.
             code = self.stack.pop()
             for c in code.co_consts:
                 if isinstance(c, types.CodeType):
                     self.stack.append(c)
-            return code
-
-        raise StopIteration
-
-    next = __next__
+            yield code

python/helpers/coverage/cmdline.py

@@ -1,10 +1,14 @@
 """Command-line support for Coverage."""
 
-import optparse, re, sys, traceback
-
+import optparse
+import os
+import sys
+import time
+import traceback
 from coverage.backward import sorted                # pylint: disable=W0622
 from coverage.execfile import run_python_file, run_python_module
 from coverage.misc import CoverageException, ExceptionDuringRun, NoSource
+from coverage.debug import info_formatter
 
 
 class Opts(object):
@@ -19,11 +23,18 @@ class Opts(object):
         '', '--branch', action='store_true',
         help="Measure branch coverage in addition to statement coverage."
         )
+    debug = optparse.make_option(
+        '', '--debug', action='store', metavar="OPTS",
+        help="Debug options, separated by commas"
+        )
     directory = optparse.make_option(
-        '-d', '--directory', action='store',
-        metavar="DIR",
+        '-d', '--directory', action='store', metavar="DIR",
         help="Write the output files to DIR."
         )
+    fail_under = optparse.make_option(
+        '', '--fail-under', action='store', metavar="MIN", type="int",
+        help="Exit with a status of 2 if the total coverage is less than MIN."
+        )
     help = optparse.make_option(
         '-h', '--help', action='store_true',
         help="Get help on this command."
@@ -89,6 +100,10 @@ class Opts(object):
         help="Use a simpler but slower trace method.  Try this if you get "
                 "seemingly impossible results!"
         )
+    title = optparse.make_option(
+        '', '--title', action='store', metavar="TITLE",
+        help="A text string to use as the title on the HTML."
+        )
     version = optparse.make_option(
         '', '--version', action='store_true',
         help="Display version information and exit."
@@ -110,7 +125,9 @@ class CoverageOptionParser(optparse.OptionParser, object):
         self.set_defaults(
             actions=[],
             branch=None,
+            debug=None,
             directory=None,
+            fail_under=None,
             help=None,
             ignore_errors=None,
             include=None,
@@ -122,6 +139,7 @@ class CoverageOptionParser(optparse.OptionParser, object):
             show_missing=None,
             source=None,
             timid=None,
+            title=None,
             erase_first=None,
             version=None,
             )
@@ -273,9 +291,11 @@ CMDS = {
     'html': CmdOptionParser("html",
         [
             Opts.directory,
+            Opts.fail_under,
             Opts.ignore_errors,
             Opts.omit,
             Opts.include,
+            Opts.title,
             ] + GLOBAL_ARGS,
         usage = "[options] [modules]",
         description = "Create an HTML report of the coverage of the files.  "
@@ -285,6 +305,7 @@ CMDS = {
 
     'report': CmdOptionParser("report",
         [
+            Opts.fail_under,
             Opts.ignore_errors,
             Opts.omit,
             Opts.include,
@@ -298,6 +319,7 @@ CMDS = {
         [
             Opts.append,
             Opts.branch,
+            Opts.debug,
             Opts.pylib,
             Opts.parallel_mode,
             Opts.module,
@@ -314,20 +336,20 @@ CMDS = {
 
     'xml': CmdOptionParser("xml",
         [
+            Opts.fail_under,
             Opts.ignore_errors,
             Opts.omit,
             Opts.include,
             Opts.output_xml,
             ] + GLOBAL_ARGS,
         cmd = "xml",
-        defaults = {'outfile': 'coverage.xml'},
         usage = "[options] [modules]",
         description = "Generate an XML report of coverage results."
         ),
     }
 
 
-OK, ERR = 0, 1
+OK, ERR, FAIL_UNDER = 0, 1, 2
 
 
 class CoverageScript(object):
@@ -346,27 +368,10 @@ class CoverageScript(object):
         self.run_python_file = _run_python_file or run_python_file
         self.run_python_module = _run_python_module or run_python_module
         self.help_fn = _help_fn or self.help
+        self.classic = False
 
         self.coverage = None
 
-    def help(self, error=None, topic=None, parser=None):
-        """Display an error message, or the named topic."""
-        assert error or topic or parser
-        if error:
-            print(error)
-            print("Use 'coverage help' for help.")
-        elif parser:
-            print(parser.format_help().strip())
-        else:
-            # Parse out the topic we want from HELP_TOPICS
-            topic_list = re.split("(?m)^=+ (\w+) =+$", HELP_TOPICS)
-            topics = dict(zip(topic_list[1::2], topic_list[2::2]))
-            help_msg = topics.get(topic, '').strip()
-            if help_msg:
-                print(help_msg % self.covpkg.__dict__)
-            else:
-                print("Don't know topic %r" % topic)
-
     def command_line(self, argv):
         """The bulk of the command line interface to Coverage.
 
@@ -376,15 +381,14 @@ class CoverageScript(object):
 
         """
         # Collect the command-line options.
-
         if not argv:
             self.help_fn(topic='minimum_help')
             return OK
 
         # The command syntax we parse depends on the first argument.  Classic
         # syntax always starts with an option.
-        classic = argv[0].startswith('-')
-        if classic:
+        self.classic = argv[0].startswith('-')
+        if self.classic:
             parser = ClassicOptionParser()
         else:
             parser = CMDS.get(argv[0])
@@ -398,64 +402,19 @@ class CoverageScript(object):
         if not ok:
             return ERR
 
-        # Handle help.
-        if options.help:
-            if classic:
-                self.help_fn(topic='help')
-            else:
-                self.help_fn(parser=parser)
-            return OK
-
-        if "help" in options.actions:
-            if args:
-                for a in args:
-                    parser = CMDS.get(a)
-                    if parser:
-                        self.help_fn(parser=parser)
-                    else:
-                        self.help_fn(topic=a)
-            else:
-                self.help_fn(topic='help')
-            return OK
-
-        # Handle version.
-        if options.version:
-            self.help_fn(topic='version')
+        # Handle help and version.
+        if self.do_help(options, args, parser):
             return OK
 
         # Check for conflicts and problems in the options.
-        for i in ['erase', 'execute']:
-            for j in ['annotate', 'html', 'report', 'combine']:
-                if (i in options.actions) and (j in options.actions):
-                    self.help_fn("You can't specify the '%s' and '%s' "
-                              "options at the same time." % (i, j))
-                    return ERR
-
-        if not options.actions:
-            self.help_fn(
-                "You must specify at least one of -e, -x, -c, -r, -a, or -b."
-                )
-            return ERR
-        args_allowed = (
-            'execute' in options.actions or
-            'annotate' in options.actions or
-            'html' in options.actions or
-            'debug' in options.actions or
-            'report' in options.actions or
-            'xml' in options.actions
-            )
-        if not args_allowed and args:
-            self.help_fn("Unexpected arguments: %s" % " ".join(args))
-            return ERR
-
-        if 'execute' in options.actions and not args:
-            self.help_fn("Nothing to do.")
+        if not self.args_ok(options, args):
             return ERR
 
         # Listify the list options.
         source = unshell_list(options.source)
         omit = unshell_list(options.omit)
         include = unshell_list(options.include)
+        debug = unshell_list(options.debug)
 
         # Do something.
         self.coverage = self.covpkg.coverage(
@@ -467,41 +426,11 @@ class CoverageScript(object):
             source = source,
             omit = omit,
             include = include,
+            debug = debug,
             )
 
         if 'debug' in options.actions:
-            if not args:
-                self.help_fn("What information would you like: data, sys?")
-                return ERR
-            for info in args:
-                if info == 'sys':
-                    print("-- sys ----------------------------------------")
-                    for label, info in self.coverage.sysinfo():
-                        if info == []:
-                            info = "-none-"
-                        if isinstance(info, list):
-                            print("%15s:" % label)
-                            for e in info:
-                                print("%15s  %s" % ("", e))
-                        else:
-                            print("%15s: %s" % (label, info))
-                elif info == 'data':
-                    print("-- data ---------------------------------------")
-                    self.coverage.load()
-                    print("path: %s" % self.coverage.data.filename)
-                    print("has_arcs: %r" % self.coverage.data.has_arcs())
-                    summary = self.coverage.data.summary(fullpath=True)
-                    if summary:
-                        filenames = sorted(summary.keys())
-                        print("\n%d files:" % len(filenames))
-                        for f in filenames:
-                            print("%s: %d lines" % (f, summary[f]))
-                    else:
-                        print("No data collected")
-                else:
-                    self.help_fn("Don't know what you mean by %r" % info)
-                    return ERR
-            return OK
+            return self.do_debug(args)
 
         if 'erase' in options.actions or options.erase_first:
             self.coverage.erase()
@@ -509,22 +438,7 @@ class CoverageScript(object):
             self.coverage.load()
 
         if 'execute' in options.actions:
-            # Run the script.
-            self.coverage.start()
-            code_ran = True
-            try:
-                try:
-                    if options.module:
-                        self.run_python_module(args[0], args)
-                    else:
-                        self.run_python_file(args[0], args)
-                except NoSource:
-                    code_ran = False
-                    raise
-            finally:
-                if code_ran:
-                    self.coverage.stop()
-                    self.coverage.save()
+            self.do_execute(options, args)
 
         if 'combine' in options.actions:
             self.coverage.combine()
@@ -539,18 +453,167 @@ class CoverageScript(object):
             )
 
         if 'report' in options.actions:
-            self.coverage.report(
+            total = self.coverage.report(
                 show_missing=options.show_missing, **report_args)
         if 'annotate' in options.actions:
             self.coverage.annotate(
                 directory=options.directory, **report_args)
         if 'html' in options.actions:
-            self.coverage.html_report(
-                directory=options.directory, **report_args)
+            total = self.coverage.html_report(
+                directory=options.directory, title=options.title,
+                **report_args)
         if 'xml' in options.actions:
             outfile = options.outfile
-            self.coverage.xml_report(outfile=outfile, **report_args)
+            total = self.coverage.xml_report(outfile=outfile, **report_args)
 
+        if options.fail_under is not None:
+            if total >= options.fail_under:
+                return OK
+            else:
+                return FAIL_UNDER
+        else:
+            return OK
+
+    def help(self, error=None, topic=None, parser=None):
+        """Display an error message, or the named topic."""
+        assert error or topic or parser
+        if error:
+            print(error)
+            print("Use 'coverage help' for help.")
+        elif parser:
+            print(parser.format_help().strip())
+        else:
+            help_msg = HELP_TOPICS.get(topic, '').strip()
+            if help_msg:
+                print(help_msg % self.covpkg.__dict__)
+            else:
+                print("Don't know topic %r" % topic)
+
+    def do_help(self, options, args, parser):
+        """Deal with help requests.
+
+        Return True if it handled the request, False if not.
+
+        """
+        # Handle help.
+        if options.help:
+            if self.classic:
+                self.help_fn(topic='help')
+            else:
+                self.help_fn(parser=parser)
+            return True
+
+        if "help" in options.actions:
+            if args:
+                for a in args:
+                    parser = CMDS.get(a)
+                    if parser:
+                        self.help_fn(parser=parser)
+                    else:
+                        self.help_fn(topic=a)
+            else:
+                self.help_fn(topic='help')
+            return True
+
+        # Handle version.
+        if options.version:
+            self.help_fn(topic='version')
+            return True
+
+        return False
+
+    def args_ok(self, options, args):
+        """Check for conflicts and problems in the options.
+
+        Returns True if everything is ok, or False if not.
+
+        """
+        for i in ['erase', 'execute']:
+            for j in ['annotate', 'html', 'report', 'combine']:
+                if (i in options.actions) and (j in options.actions):
+                    self.help_fn("You can't specify the '%s' and '%s' "
+                              "options at the same time." % (i, j))
+                    return False
+
+        if not options.actions:
+            self.help_fn(
+                "You must specify at least one of -e, -x, -c, -r, -a, or -b."
+                )
+            return False
+        args_allowed = (
+            'execute' in options.actions or
+            'annotate' in options.actions or
+            'html' in options.actions or
+            'debug' in options.actions or
+            'report' in options.actions or
+            'xml' in options.actions
+            )
+        if not args_allowed and args:
+            self.help_fn("Unexpected arguments: %s" % " ".join(args))
+            return False
+
+        if 'execute' in options.actions and not args:
+            self.help_fn("Nothing to do.")
+            return False
+
+        return True
+
+    def do_execute(self, options, args):
+        """Implementation of 'coverage run'."""
+
+        # Set the first path element properly.
+        old_path0 = sys.path[0]
+
+        # Run the script.
+        self.coverage.start()
+        code_ran = True
+        try:
+            try:
+                if options.module:
+                    sys.path[0] = ''
+                    self.run_python_module(args[0], args)
+                else:
+                    filename = args[0]
+                    sys.path[0] = os.path.abspath(os.path.dirname(filename))
+                    self.run_python_file(filename, args)
+            except NoSource:
+                code_ran = False
+                raise
+        finally:
+            self.coverage.stop()
+            if code_ran:
+                self.coverage.save()
+
+            # Restore the old path
+            sys.path[0] = old_path0
+
+    def do_debug(self, args):
+        """Implementation of 'coverage debug'."""
+
+        if not args:
+            self.help_fn("What information would you like: data, sys?")
+            return ERR
+        for info in args:
+            if info == 'sys':
+                print("-- sys ----------------------------------------")
+                for line in info_formatter(self.coverage.sysinfo()):
+                    print(" %s" % line)
+            elif info == 'data':
+                print("-- data ---------------------------------------")
+                self.coverage.load()
+                print("path: %s" % self.coverage.data.filename)
+                print("has_arcs: %r" % self.coverage.data.has_arcs())
+                summary = self.coverage.data.summary(fullpath=True)
+                if summary:
+                    filenames = sorted(summary.keys())
+                    print("\n%d files:" % len(filenames))
+                    for f in filenames:
+                        print("%s: %d lines" % (f, summary[f]))
+                else:
+                    print("No data collected")
+            else:
+                self.help_fn("Don't know what you mean by %r" % info)
+                return ERR
         return OK
 
 
@@ -568,10 +631,10 @@ def unshell_list(s):
     return s.split(',')
 
 
-HELP_TOPICS = r"""
-
-== classic ====================================================================
-Coverage.py version %(__version__)s
+HELP_TOPICS = {
+# -------------------------
+'classic':
+r"""Coverage.py version %(__version__)s
 Measure, collect, and report on code coverage in Python programs.
 
 Usage:
@@ -615,8 +678,9 @@ coverage -a [-d DIR] [-i] [-o DIR,...] [FILE1 FILE2 ...]
 
 Coverage data is saved in the file .coverage by default.  Set the
 COVERAGE_FILE environment variable to save it somewhere else.
-
-== help =======================================================================
+""",
+# -------------------------
+'help': """\
 Coverage.py, version %(__version__)s
 Measure, collect, and report on code coverage in Python programs.
 
@@ -635,26 +699,32 @@ Commands:
 Use "coverage help <command>" for detailed help on any command.
 Use "coverage help classic" for help on older command syntax.
 For more information, see %(__url__)s
-
-== minimum_help ===============================================================
+""",
+# -------------------------
+'minimum_help': """\
 Code coverage for Python.  Use 'coverage help' for help.
-
-== version ====================================================================
+""",
+# -------------------------
+'version': """\
 Coverage.py, version %(__version__)s.  %(__url__)s
-
-"""
+""",
+}
 
 
 def main(argv=None):
-    """The main entrypoint to Coverage.
+    """The main entry point to Coverage.
 
-    This is installed as the script entrypoint.
+    This is installed as the script entry point.
 
     """
     if argv is None:
         argv = sys.argv[1:]
     try:
+        start = time.clock()
         status = CoverageScript().command_line(argv)
+        end = time.clock()
+        if 0:
+            print("time: %.3fs" % (end - start))
     except ExceptionDuringRun:
         # An exception was caught while running the product code.  The
         # sys.exc_info() return tuple is packed into an ExceptionDuringRun

python/helpers/coverage/codeunit.py

@@ -1,7 +1,7 @@
 """Code unit (module) handling for Coverage."""
 
-import glob, os
-
+import glob
+import os
 from coverage.backward import open_source, string_class, StringIO
 from coverage.misc import CoverageException
 
@@ -52,8 +52,10 @@ class CodeUnit(object):
         else:
             f = morf
         # .pyc files should always refer to a .py instead.
-        if f.endswith('.pyc'):
+        if f.endswith('.pyc') or f.endswith('.pyo'):
             f = f[:-1]
+        elif f.endswith('$py.class'): # Jython
+            f = f[:-9] + ".py"
         self.filename = self.file_locator.canonical_filename(f)
 
         if hasattr(morf, '__name__'):
@@ -77,12 +79,18 @@ class CodeUnit(object):
     # Annoying comparison operators. Py3k wants __lt__ etc, and Py2k needs all
     # of them defined.
 
-    def __lt__(self, other): return self.name <  other.name
-    def __le__(self, other): return self.name <= other.name
-    def __eq__(self, other): return self.name == other.name
-    def __ne__(self, other): return self.name != other.name
-    def __gt__(self, other): return self.name >  other.name
-    def __ge__(self, other): return self.name >= other.name
+    def __lt__(self, other):
+        return self.name < other.name
+    def __le__(self, other):
+        return self.name <= other.name
+    def __eq__(self, other):
+        return self.name == other.name
+    def __ne__(self, other):
+        return self.name != other.name
+    def __gt__(self, other):
+        return self.name > other.name
+    def __ge__(self, other):
+        return self.name >= other.name
 
     def flat_rootname(self):
         """A base for a flat filename to correspond to this code unit.
@@ -113,5 +121,25 @@ class CodeUnit(object):
 
         # Couldn't find source.
         raise CoverageException(
-            "No source for code %r." % self.filename
+            "No source for code '%s'." % self.filename
             )
+
+    def should_be_python(self):
+        """Does it seem like this file should contain Python?
+
+        This is used to decide if a file reported as part of the exection of
+        a program was really likely to have contained Python in the first
+        place.
+
+        """
+        # Get the file extension.
+        _, ext = os.path.splitext(self.filename)
+
+        # Anything named *.py* should be Python.
+        if ext.startswith('.py'):
+            return True
+        # A file with no extension should be Python.
+        if not ext:
+            return True
+        # Everything else is probably not Python.
+        return False

python/helpers/coverage/collector.py

@@ -1,13 +1,26 @@
 """Raw data collector for Coverage."""
 
-import sys, threading
+import os
+import sys
+import threading
 
 try:
     # Use the C extension code when we can, for speed.
-    from coverage.tracer import Tracer
+    from coverage.tracer import CTracer         # pylint: disable=F0401,E0611
 except ImportError:
     # Couldn't import the C extension, maybe it isn't built.
-    Tracer = None
+    if os.getenv('COVERAGE_TEST_TRACER') == 'c':
+        # During testing, we use the COVERAGE_TEST_TRACER env var to indicate
+        # that we've fiddled with the environment to test this fallback code.
+        # If we thought we had a C tracer, but couldn't import it, then exit
+        # quickly and clearly instead of dribbling confusing errors. I'm using
+        # sys.exit here instead of an exception because an exception here
+        # causes all sorts of other noise in unittest.
+        sys.stderr.write(
+            "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n"
+            )
+        sys.exit(1)
+    CTracer = None
 
 
 class PyTracer(object):
@@ -40,12 +53,19 @@ class PyTracer(object):
         self.last_exc_back = None
         self.last_exc_firstlineno = 0
         self.arcs = False
+        self.thread = None
+        self.stopped = False
 
     def _trace(self, frame, event, arg_unused):
         """The trace function passed to sys.settrace."""
 
-        #print("trace event: %s %r @%d" % (
-        #           event, frame.f_code.co_filename, frame.f_lineno))
+        if self.stopped:
+            return
+
+        if 0:
+            sys.stderr.write("trace event: %s %r @%d\n" % (
+                event, frame.f_code.co_filename, frame.f_lineno
+            ))
 
         if self.last_exc_back:
             if frame == self.last_exc_back:
@@ -61,10 +81,11 @@ class PyTracer(object):
             # in this file.
             self.data_stack.append((self.cur_file_data, self.last_line))
             filename = frame.f_code.co_filename
-            tracename = self.should_trace_cache.get(filename)
-            if tracename is None:
+            if filename not in self.should_trace_cache:
                 tracename = self.should_trace(filename, frame)
                 self.should_trace_cache[filename] = tracename
+            else:
+                tracename = self.should_trace_cache[filename]
             #print("called, stack is %d deep, tracename is %r" % (
             #               len(self.data_stack), tracename))
             if tracename:
@@ -105,15 +126,24 @@ class PyTracer(object):
         Return a Python function suitable for use with sys.settrace().
 
         """
+        self.thread = threading.currentThread()
         sys.settrace(self._trace)
         return self._trace
 
     def stop(self):
         """Stop this Tracer."""
+        self.stopped = True
+        if self.thread != threading.currentThread():
+            # Called on a different thread than started us: we can't unhook
+            # ourseves, but we've set the flag that we should stop, so we won't
+            # do any more tracing.
+            return
+
         if hasattr(sys, "gettrace") and self.warn:
             if sys.gettrace() != self._trace:
                 msg = "Trace function changed, measurement is likely wrong: %r"
-                self.warn(msg % sys.gettrace())
+                self.warn(msg % (sys.gettrace(),))
+        #print("Stopping tracer on %s" % threading.current_thread().ident)
         sys.settrace(None)
 
     def get_stats(self):
@@ -146,7 +176,7 @@ class Collector(object):
         """Create a collector.
 
         `should_trace` is a function, taking a filename, and returning a
-        canonicalized filename, or False depending on whether the file should
+        canonicalized filename, or None depending on whether the file should
         be traced or not.
 
         If `timid` is true, then a slower simpler trace function will be
@@ -173,7 +203,7 @@ class Collector(object):
         else:
             # Being fast: use the C Tracer if it is available, else the Python
             # trace function.
-            self._trace_class = Tracer or PyTracer
+            self._trace_class = CTracer or PyTracer
 
     def __repr__(self):
         return "<Collector at 0x%x>" % id(self)
@@ -190,7 +220,7 @@ class Collector(object):
 
         # A cache of the results from should_trace, the decision about whether
         # to trace execution in a file. A dict of filename to (filename or
-        # False).
+        # None).
         self.should_trace_cache = {}
 
         # Our active Tracers.
@@ -232,9 +262,29 @@ class Collector(object):
         if self._collectors:
             self._collectors[-1].pause()
         self._collectors.append(self)
-        #print >>sys.stderr, "Started: %r" % self._collectors
+        #print("Started: %r" % self._collectors, file=sys.stderr)
+
+        # Check to see whether we had a fullcoverage tracer installed.
+        traces0 = []
+        if hasattr(sys, "gettrace"):
+            fn0 = sys.gettrace()
+            if fn0:
+                tracer0 = getattr(fn0, '__self__', None)
+                if tracer0:
+                    traces0 = getattr(tracer0, 'traces', [])
+
         # Install the tracer on this thread.
-        self._start_tracer()
+        fn = self._start_tracer()
+
+        for args in traces0:
+            (frame, event, arg), lineno = args
+            try:
+                fn(frame, event, arg, lineno=lineno)
+            except TypeError:
+                raise Exception(
+                    "fullcoverage must be run with the C trace function."
+                )
+
         # Install our installation tracer in threading, to jump start other
         # threads.
         threading.settrace(self._installation_trace)

python/helpers/coverage/config.py

@@ -1,7 +1,79 @@
 """Config file for coverage.py"""
 
 import os
-from coverage.backward import configparser          # pylint: disable=W0622
+import re
+import sys
+from coverage.backward import string_class, iitems
+
+
+# In py3, # ConfigParser was renamed to the more-standard configparser
+try:
+    import configparser                             # pylint: disable=F0401
+except ImportError:
+    import ConfigParser as configparser
+
+
+class HandyConfigParser(configparser.RawConfigParser):
+    """Our specialization of ConfigParser."""
+
+    def read(self, filename):
+        """Read a filename as UTF-8 configuration data."""
+        kwargs = {}
+        if sys.version_info >= (3, 2):
+            kwargs['encoding'] = "utf-8"
+        return configparser.RawConfigParser.read(self, filename, **kwargs)
+
+    def get(self, *args, **kwargs):
+        v = configparser.RawConfigParser.get(self, *args, **kwargs)
+        def dollar_replace(m):
+            """Called for each $replacement."""
+            # Only one of the groups will have matched, just get its text.
+            word = [w for w in m.groups() if w is not None][0]
+            if word == "$":
+                return "$"
+            else:
+                return os.environ.get(word, '')
+
+        dollar_pattern = r"""(?x)   # Use extended regex syntax
+            \$(?:                   # A dollar sign, then
+            (?P<v1>\w+) |           #   a plain word,
+            {(?P<v2>\w+)} |         #   or a {-wrapped word,
+            (?P<char>[$])           #   or a dollar sign.
+            )
+            """
+        v = re.sub(dollar_pattern, dollar_replace, v)
+        return v
+
+    def getlist(self, section, option):
+        """Read a list of strings.
+
+        The value of `section` and `option` is treated as a comma- and newline-
+        separated list of strings.  Each value is stripped of whitespace.
+
+        Returns the list of strings.
+
+        """
+        value_list = self.get(section, option)
+        values = []
+        for value_line in value_list.split('\n'):
+            for value in value_line.split(','):
+                value = value.strip()
+                if value:
+                    values.append(value)
+        return values
+
+    def getlinelist(self, section, option):
+        """Read a list of full-line strings.
+
+        The value of `section` and `option` is treated as a newline-separated
+        list of strings.  Each value is stripped of whitespace.
+
+        Returns the list of strings.
+
+        """
+        value_list = self.get(section, option)
+        return list(filter(None, value_list.split('\n')))
+
 
 # The default line exclusion regexes
 DEFAULT_EXCLUDE = [
@@ -29,9 +101,12 @@ class CoverageConfig(object):
     operation of coverage.py.
 
     """
-
     def __init__(self):
         """Initialize the configuration attributes to their defaults."""
+        # Metadata about the config.
+        self.attempted_config_files = []
+        self.config_files = []
+
         # Defaults for [run]
         self.branch = False
         self.cover_pylib = False
@@ -39,6 +114,7 @@ class CoverageConfig(object):
         self.parallel = False
         self.timid = False
         self.source = None
+        self.debug = []
 
         # Defaults for [report]
         self.exclude_list = DEFAULT_EXCLUDE[:]
@@ -48,13 +124,19 @@ class CoverageConfig(object):
         self.partial_list = DEFAULT_PARTIAL[:]
         self.partial_always_list = DEFAULT_PARTIAL_ALWAYS[:]
         self.precision = 0
+        self.show_missing = False
 
         # Defaults for [html]
         self.html_dir = "htmlcov"
+        self.extra_css = None
+        self.html_title = "Coverage report"
 
         # Defaults for [xml]
         self.xml_output = "coverage.xml"
 
+        # Defaults for [paths]
+        self.paths = {}
+
     def from_environment(self, env_var):
         """Read configuration from the `env_var` environment variable."""
         # Timidity: for nose users, read an environment variable.  This is a
@@ -64,93 +146,71 @@ class CoverageConfig(object):
         if env:
             self.timid = ('--timid' in env)
 
+    MUST_BE_LIST = ["omit", "include", "debug"]
+
     def from_args(self, **kwargs):
         """Read config values from `kwargs`."""
-        for k, v in kwargs.items():
+        for k, v in iitems(kwargs):
             if v is not None:
+                if k in self.MUST_BE_LIST and isinstance(v, string_class):
+                    v = [v]
                 setattr(self, k, v)
 
-    def from_file(self, *files):
-        """Read configuration from .rc files.
+    def from_file(self, filename):
+        """Read configuration from a .rc file.
 
-        Each argument in `files` is a file name to read.
+        `filename` is a file name to read.
 
         """
-        cp = configparser.RawConfigParser()
-        cp.read(files)
+        self.attempted_config_files.append(filename)
 
+        cp = HandyConfigParser()
+        files_read = cp.read(filename)
+        if files_read is not None:  # return value changed in 2.4
+            self.config_files.extend(files_read)
+
+        for option_spec in self.CONFIG_FILE_OPTIONS:
+            self.set_attr_from_config_option(cp, *option_spec)
+
+        # [paths] is special
+        if cp.has_section('paths'):
+            for option in cp.options('paths'):
+                self.paths[option] = cp.getlist('paths', option)
+
+    CONFIG_FILE_OPTIONS = [
         # [run]
-        if cp.has_option('run', 'branch'):
-            self.branch = cp.getboolean('run', 'branch')
-        if cp.has_option('run', 'cover_pylib'):
-            self.cover_pylib = cp.getboolean('run', 'cover_pylib')
-        if cp.has_option('run', 'data_file'):
-            self.data_file = cp.get('run', 'data_file')
-        if cp.has_option('run', 'include'):
-            self.include = self.get_list(cp, 'run', 'include')
-        if cp.has_option('run', 'omit'):
-            self.omit = self.get_list(cp, 'run', 'omit')
-        if cp.has_option('run', 'parallel'):
-            self.parallel = cp.getboolean('run', 'parallel')
-        if cp.has_option('run', 'source'):
-            self.source = self.get_list(cp, 'run', 'source')
-        if cp.has_option('run', 'timid'):
-            self.timid = cp.getboolean('run', 'timid')
+        ('branch', 'run:branch', 'boolean'),
+        ('cover_pylib', 'run:cover_pylib', 'boolean'),
+        ('data_file', 'run:data_file'),
+        ('debug', 'run:debug', 'list'),
+        ('include', 'run:include', 'list'),
+        ('omit', 'run:omit', 'list'),
+        ('parallel', 'run:parallel', 'boolean'),
+        ('source', 'run:source', 'list'),
+        ('timid', 'run:timid', 'boolean'),
 
         # [report]
-        if cp.has_option('report', 'exclude_lines'):
-            self.exclude_list = \
-                self.get_line_list(cp, 'report', 'exclude_lines')
-        if cp.has_option('report', 'ignore_errors'):
-            self.ignore_errors = cp.getboolean('report', 'ignore_errors')
-        if cp.has_option('report', 'include'):
-            self.include = self.get_list(cp, 'report', 'include')
-        if cp.has_option('report', 'omit'):
-            self.omit = self.get_list(cp, 'report', 'omit')
-        if cp.has_option('report', 'partial_branches'):
-            self.partial_list = \
-                self.get_line_list(cp, 'report', 'partial_branches')
-        if cp.has_option('report', 'partial_branches_always'):
-            self.partial_always_list = \
-                self.get_line_list(cp, 'report', 'partial_branches_always')
-        if cp.has_option('report', 'precision'):
-            self.precision = cp.getint('report', 'precision')
+        ('exclude_list', 'report:exclude_lines', 'linelist'),
+        ('ignore_errors', 'report:ignore_errors', 'boolean'),
+        ('include', 'report:include', 'list'),
+        ('omit', 'report:omit', 'list'),
+        ('partial_list', 'report:partial_branches', 'linelist'),
+        ('partial_always_list', 'report:partial_branches_always', 'linelist'),
+        ('precision', 'report:precision', 'int'),
+        ('show_missing', 'report:show_missing', 'boolean'),
 
         # [html]
-        if cp.has_option('html', 'directory'):
-            self.html_dir = cp.get('html', 'directory')
+        ('html_dir', 'html:directory'),
+        ('extra_css', 'html:extra_css'),
+        ('html_title', 'html:title'),
 
         # [xml]
-        if cp.has_option('xml', 'output'):
-            self.xml_output = cp.get('xml', 'output')
-
-    def get_list(self, cp, section, option):
-        """Read a list of strings from the ConfigParser `cp`.
-
-        The value of `section` and `option` is treated as a comma- and newline-
-        separated list of strings.  Each value is stripped of whitespace.
-
-        Returns the list of strings.
-
-        """
-        value_list = cp.get(section, option)
-        values = []
-        for value_line in value_list.split('\n'):
-            for value in value_line.split(','):
-                value = value.strip()
-                if value:
-                    values.append(value)
-        return values
-
-    def get_line_list(self, cp, section, option):
-        """Read a list of full-line strings from the ConfigParser `cp`.
-
-        The value of `section` and `option` is treated as a newline-separated
-        list of strings.  Each value is stripped of whitespace.
-
-        Returns the list of strings.
-
-        """
-        value_list = cp.get(section, option)
-        return list(filter(None, value_list.split('\n')))
+        ('xml_output', 'xml:output'),
+        ]
 
+    def set_attr_from_config_option(self, cp, attr, where, type_=''):
+        """Set an attribute on self if it exists in the ConfigParser."""
+        section, option = where.split(":")
+        if cp.has_option(section, option):
+            method = getattr(cp, 'get'+type_)
+            setattr(self, attr, method(section, option))

python/helpers/coverage/control.py

@@ -1,23 +1,36 @@
 """Core control stuff for Coverage."""
 
-import atexit, os, random, socket, sys
-
+import atexit
+import os
+import random
+import socket
+import sys
 from coverage.annotate import AnnotateReporter
-from coverage.backward import string_class
+from coverage.backward import string_class, iitems, sorted  # pylint: disable=W0622
 from coverage.codeunit import code_unit_factory, CodeUnit
 from coverage.collector import Collector
 from coverage.config import CoverageConfig
 from coverage.data import CoverageData
+from coverage.debug import DebugControl
 from coverage.files import FileLocator, TreeMatcher, FnmatchMatcher
-from coverage.files import find_python_files
+from coverage.files import PathAliases, find_python_files, prep_patterns
 from coverage.html import HtmlReporter
 from coverage.misc import CoverageException, bool_or_none, join_regex
+from coverage.misc import file_be_gone
 from coverage.results import Analysis, Numbers
 from coverage.summary import SummaryReporter
 from coverage.xmlreport import XmlReporter
 
+# Pypy has some unusual stuff in the "stdlib".  Consider those locations
+# when deciding where the stdlib is.
+try:
+    import _structseq       # pylint: disable=F0401
+except ImportError:
+    _structseq = None
+
+
 class coverage(object):
-    """Programmatic access to Coverage.
+    """Programmatic access to coverage.py.
 
     To use::
 
@@ -25,14 +38,15 @@ class coverage(object):
 
         cov = coverage()
         cov.start()
-        #.. blah blah (run your code) blah blah ..
+        #.. call your code ..
         cov.stop()
         cov.html_report(directory='covhtml')
 
     """
     def __init__(self, data_file=None, data_suffix=None, cover_pylib=None,
                 auto_data=False, timid=None, branch=None, config_file=True,
-                source=None, omit=None, include=None):
+                source=None, omit=None, include=None, debug=None,
+                debug_file=None):
         """
         `data_file` is the base name of the data file to use, defaulting to
         ".coverage".  `data_suffix` is appended (with a dot) to `data_file` to
@@ -67,6 +81,10 @@ class coverage(object):
         `include` will be measured, files that match `omit` will not.  Each
         will also accept a single string argument.
 
+        `debug` is a list of strings indicating what debugging information is
+        desired. `debug_file` is the file to write debug messages to,
+        defaulting to stderr.
+
         """
         from coverage import __version__
 
@@ -96,18 +114,16 @@ class coverage(object):
             self.config.data_file = env_data_file
 
         # 4: from constructor arguments:
-        if isinstance(omit, string_class):
-            omit = [omit]
-        if isinstance(include, string_class):
-            include = [include]
         self.config.from_args(
             data_file=data_file, cover_pylib=cover_pylib, timid=timid,
             branch=branch, parallel=bool_or_none(data_suffix),
-            source=source, omit=omit, include=include
+            source=source, omit=omit, include=include, debug=debug,
             )
 
+        # Create and configure the debugging controller.
+        self.debug = DebugControl(self.config.debug, debug_file or sys.stderr)
+
         self.auto_data = auto_data
-        self.atexit_registered = False
 
         # _exclude_re is a dict mapping exclusion list names to compiled
         # regexes.
@@ -125,8 +141,8 @@ class coverage(object):
             else:
                 self.source_pkgs.append(src)
 
-        self.omit = self._prep_patterns(self.config.omit)
-        self.include = self._prep_patterns(self.config.include)
+        self.omit = prep_patterns(self.config.omit)
+        self.include = prep_patterns(self.config.include)
 
         self.collector = Collector(
             self._should_trace, timid=self.config.timid,
@@ -151,7 +167,8 @@ class coverage(object):
         # started rather than wherever the process eventually chdir'd to.
         self.data = CoverageData(
             basename=self.config.data_file,
-            collector="coverage v%s" % __version__
+            collector="coverage v%s" % __version__,
+            debug=self.debug,
             )
 
         # The dirs for files considered "installed with the interpreter".
@@ -162,9 +179,9 @@ class coverage(object):
             # environments (virtualenv, for example), these modules may be
             # spread across a few locations. Look at all the candidate modules
             # we've imported, and take all the different ones.
-            for m in (atexit, os, random, socket):
-                if hasattr(m, "__file__"):
-                    m_dir = self._canonical_dir(m.__file__)
+            for m in (atexit, os, random, socket, _structseq):
+                if m is not None and hasattr(m, "__file__"):
+                    m_dir = self._canonical_dir(m)
                     if m_dir not in self.pylib_dirs:
                         self.pylib_dirs.append(m_dir)
 
@@ -172,63 +189,61 @@ class coverage(object):
         # where we are.
         self.cover_dir = self._canonical_dir(__file__)
 
-        # The matchers for _should_trace, created when tracing starts.
+        # The matchers for _should_trace.
         self.source_match = None
         self.pylib_match = self.cover_match = None
         self.include_match = self.omit_match = None
 
-        # Only _harvest_data once per measurement cycle.
-        self._harvested = False
-
         # Set the reporting precision.
         Numbers.set_precision(self.config.precision)
 
-        # When tearing down the coverage object, modules can become None.
-        # Saving the modules as object attributes avoids problems, but it is
-        # quite ad-hoc which modules need to be saved and which references
-        # need to use the object attributes.
-        self.socket = socket
-        self.os = os
-        self.random = random
+        # Is it ok for no data to be collected?
+        self._warn_no_data = True
+        self._warn_unimported_source = True
 
-    def _canonical_dir(self, f):
-        """Return the canonical directory of the file `f`."""
-        return os.path.split(self.file_locator.canonical_filename(f))[0]
+        # State machine variables:
+        # Have we started collecting and not stopped it?
+        self._started = False
+        # Have we measured some data and not harvested it?
+        self._measured = False
+
+        atexit.register(self._atexit)
+
+    def _canonical_dir(self, morf):
+        """Return the canonical directory of the module or file `morf`."""
+        return os.path.split(CodeUnit(morf, self.file_locator).filename)[0]
 
     def _source_for_file(self, filename):
         """Return the source file for `filename`."""
         if not filename.endswith(".py"):
             if filename[-4:-1] == ".py":
                 filename = filename[:-1]
+            elif filename.endswith("$py.class"): # jython
+                filename = filename[:-9] + ".py"
         return filename
 
-    def _should_trace(self, filename, frame):
-        """Decide whether to trace execution in `filename`
+    def _should_trace_with_reason(self, filename, frame):
+        """Decide whether to trace execution in `filename`, with a reason.
 
         This function is called from the trace function.  As each new file name
         is encountered, this function determines whether it is traced or not.
 
-        Returns a canonicalized filename if it should be traced, False if it
-        should not.
+        Returns a pair of values:  the first indicates whether the file should
+        be traced: it's a canonicalized filename if it should be traced, None
+        if it should not.  The second value is a string, the resason for the
+        decision.
 
         """
-        if os is None:
-            return False
+        if not filename:
+            # Empty string is pretty useless
+            return None, "empty string isn't a filename"
 
         if filename.startswith('<'):
             # Lots of non-file execution is represented with artificial
             # filenames like "<string>", "<doctest readme.txt[0]>", or
             # "<exec_function>".  Don't ever trace these executions, since we
             # can't do anything with the data later anyway.
-            return False
-
-        if filename.endswith(".html"):
-            # Jinja and maybe other templating systems compile templates into
-            # Python code, but use the template filename as the filename in
-            # the compiled code.  Of course, those filenames are useless later
-            # so don't bother collecting.  TODO: How should we really separate
-            # out good file extensions from bad?
-            return False
+            return None, "not a real filename"
 
         self._check_for_packages()
 
@@ -248,64 +263,53 @@ class coverage(object):
 
         canonical = self.file_locator.canonical_filename(filename)
 
-        # If the user specified source, then that's authoritative about what to
-        # measure.  If they didn't, then we have to exclude the stdlib and
-        # coverage.py directories.
+        # If the user specified source or include, then that's authoritative
+        # about the outer bound of what to measure and we don't have to apply
+        # any canned exclusions. If they didn't, then we have to exclude the
+        # stdlib and coverage.py directories.
         if self.source_match:
             if not self.source_match.match(canonical):
-                return False
+                return None, "falls outside the --source trees"
+        elif self.include_match:
+            if not self.include_match.match(canonical):
+                return None, "falls outside the --include trees"
         else:
             # If we aren't supposed to trace installed code, then check if this
             # is near the Python standard library and skip it if so.
             if self.pylib_match and self.pylib_match.match(canonical):
-                return False
+                return None, "is in the stdlib"
 
             # We exclude the coverage code itself, since a little of it will be
             # measured otherwise.
             if self.cover_match and self.cover_match.match(canonical):
-                return False
+                return None, "is part of coverage.py"
 
-        # Check the file against the include and omit patterns.
-        if self.include_match and not self.include_match.match(canonical):
-            return False
+        # Check the file against the omit pattern.
         if self.omit_match and self.omit_match.match(canonical):
-            return False
+            return None, "is inside an --omit pattern"
 
+        return canonical, "because we love you"
+
+    def _should_trace(self, filename, frame):
+        """Decide whether to trace execution in `filename`.
+
+        Calls `_should_trace_with_reason`, and returns just the decision.
+
+        """
+        canonical, reason = self._should_trace_with_reason(filename, frame)
+        if self.debug.should('trace'):
+            if not canonical:
+                msg = "Not tracing %r: %s" % (filename, reason)
+            else:
+                msg = "Tracing %r" % (filename,)
+            self.debug.write(msg)
         return canonical
 
-    # To log what should_trace returns, change this to "if 1:"
-    if 0:
-        _real_should_trace = _should_trace
-        def _should_trace(self, filename, frame):   # pylint: disable=E0102
-            """A logging decorator around the real _should_trace function."""
-            ret = self._real_should_trace(filename, frame)
-            print("should_trace: %r -> %r" % (filename, ret))
-            return ret
-
     def _warn(self, msg):
         """Use `msg` as a warning."""
         self._warnings.append(msg)
         sys.stderr.write("Coverage.py warning: %s\n" % msg)
 
-    def _prep_patterns(self, patterns):
-        """Prepare the file patterns for use in a `FnmatchMatcher`.
-
-        If a pattern starts with a wildcard, it is used as a pattern
-        as-is.  If it does not start with a wildcard, then it is made
-        absolute with the current directory.
-
-        If `patterns` is None, an empty list is returned.
-
-        """
-        patterns = patterns or []
-        prepped = []
-        for p in patterns or []:
-            if p.startswith("*") or p.startswith("?"):
-                prepped.append(p)
-            else:
-                prepped.append(self.file_locator.abs_file(p))
-        return prepped
-
     def _check_for_packages(self):
         """Update the source_match matcher with latest imported packages."""
         # Our self.source_pkgs attribute is a list of package names we want to
@@ -325,17 +329,23 @@ class coverage(object):
                 try:
                     pkg_file = mod.__file__
                 except AttributeError:
-                    self._warn("Module %s has no Python source." % pkg)
+                    pkg_file = None
                 else:
                     d, f = os.path.split(pkg_file)
-                    if f.startswith('__init__.'):
+                    if f.startswith('__init__'):
                         # This is actually a package, return the directory.
                         pkg_file = d
                     else:
                         pkg_file = self._source_for_file(pkg_file)
                     pkg_file = self.file_locator.canonical_filename(pkg_file)
+                    if not os.path.exists(pkg_file):
+                        pkg_file = None
+
+                if pkg_file:
                     self.source.append(pkg_file)
                     self.source_match.add(pkg_file)
+                else:
+                    self._warn("Module %s has no Python source." % pkg)
 
             for pkg in found:
                 self.source_pkgs.remove(pkg)
@@ -354,17 +364,21 @@ class coverage(object):
         self.data.read()
 
     def start(self):
-        """Start measuring code coverage."""
+        """Start measuring code coverage.
+
+        Coverage measurement actually occurs in functions called after `start`
+        is invoked.  Statements in the same scope as `start` won't be measured.
+
+        Once you invoke `start`, you must also call `stop` eventually, or your
+        process might not shut down cleanly.
+
+        """
         if self.run_suffix:
             # Calling start() means we're running code, so use the run_suffix
             # as the data_suffix when we eventually save the data.
             self.data_suffix = self.run_suffix
         if self.auto_data:
             self.load()
-            # Save coverage data when Python exits.
-            if not self.atexit_registered:
-                atexit.register(self.save)
-                self.atexit_registered = True
 
         # Create the matchers we need for _should_trace
         if self.source or self.source_pkgs:
@@ -379,13 +393,31 @@ class coverage(object):
         if self.omit:
             self.omit_match = FnmatchMatcher(self.omit)
 
-        self._harvested = False
+        # The user may want to debug things, show info if desired.
+        if self.debug.should('config'):
+            self.debug.write("Configuration values:")
+            config_info = sorted(self.config.__dict__.items())
+            self.debug.write_formatted_info(config_info)
+
+        if self.debug.should('sys'):
+            self.debug.write("Debugging info:")
+            self.debug.write_formatted_info(self.sysinfo())
+
         self.collector.start()
+        self._started = True
+        self._measured = True
 
     def stop(self):
         """Stop measuring code coverage."""
+        self._started = False
         self.collector.stop()
-        self._harvest_data()
+
+    def _atexit(self):
+        """Clean up on process shutdown."""
+        if self._started:
+            self.stop()
+        if self.auto_data:
+            self.save()
 
     def erase(self):
         """Erase previously-collected coverage data.
@@ -449,9 +481,15 @@ class coverage(object):
             # plenty of distinguishing information.  We do this here in
             # `save()` at the last minute so that the pid will be correct even
             # if the process forks.
-            data_suffix = "%s.%s.%06d" % (
-                self.socket.gethostname(), self.os.getpid(),
-                self.random.randint(0, 99999)
+            extra = ""
+            if _TEST_NAME_FILE:
+                f = open(_TEST_NAME_FILE)
+                test_name = f.read()
+                f.close()
+                extra = "." + test_name
+            data_suffix = "%s%s.%s.%06d" % (
+                socket.gethostname(), extra, os.getpid(),
+                random.randint(0, 999999)
                 )
 
         self._harvest_data()
@@ -465,7 +503,14 @@ class coverage(object):
         current measurements.
 
         """
-        self.data.combine_parallel_data()
+        aliases = None
+        if self.config.paths:
+            aliases = PathAliases(self.file_locator)
+            for paths in self.config.paths.values():
+                result = paths[0]
+                for pattern in paths[1:]:
+                    aliases.add(pattern, result)
+        self.data.combine_parallel_data(aliases=aliases)
 
     def _harvest_data(self):
         """Get the collected data and reset the collector.
@@ -473,27 +518,37 @@ class coverage(object):
         Also warn about various problems collecting data.
 
         """
-        if not self._harvested:
-            self.data.add_line_data(self.collector.get_line_data())
-            self.data.add_arc_data(self.collector.get_arc_data())
-            self.collector.reset()
+        if not self._measured:
+            return
 
-            # If there are still entries in the source_pkgs list, then we never
-            # encountered those packages.
+        self.data.add_line_data(self.collector.get_line_data())
+        self.data.add_arc_data(self.collector.get_arc_data())
+        self.collector.reset()
+
+        # If there are still entries in the source_pkgs list, then we never
+        # encountered those packages.
+        if self._warn_unimported_source:
             for pkg in self.source_pkgs:
                 self._warn("Module %s was never imported." % pkg)
 
-            # Find out if we got any data.
-            summary = self.data.summary()
-            if not summary:
-                self._warn("No data was collected.")
+        # Find out if we got any data.
+        summary = self.data.summary()
+        if not summary and self._warn_no_data:
+            self._warn("No data was collected.")
 
-            # Find files that were never executed at all.
-            for src in self.source:
-                for py_file in find_python_files(src):
-                    self.data.touch_file(py_file)
+        # Find files that were never executed at all.
+        for src in self.source:
+            for py_file in find_python_files(src):
+                py_file = self.file_locator.canonical_filename(py_file)
 
-            self._harvested = True
+                if self.omit_match and self.omit_match.match(py_file):
+                    # Turns out this file was omitted, so don't pull it back
+                    # in as unexecuted.
+                    continue
+
+                self.data.touch_file(py_file)
+
+        self._measured = False
 
     # Backward compatibility with version 1.
     def analysis(self, morf):
@@ -520,8 +575,11 @@ class coverage(object):
         """
         analysis = self._analyze(morf)
         return (
-            analysis.filename, analysis.statements, analysis.excluded,
-            analysis.missing, analysis.missing_formatted()
+            analysis.filename,
+            sorted(analysis.statements),
+            sorted(analysis.excluded),
+            sorted(analysis.missing),
+            analysis.missing_formatted(),
             )
 
     def _analyze(self, it):
@@ -530,6 +588,7 @@ class coverage(object):
         Returns an `Analysis` object.
 
         """
+        self._harvest_data()
         if not isinstance(it, CodeUnit):
             it = code_unit_factory(it, self.file_locator)[0]
 
@@ -548,14 +607,16 @@ class coverage(object):
         match those patterns will be included in the report. Modules matching
         `omit` will not be included in the report.
 
+        Returns a float, the total percentage covered.
+
         """
+        self._harvest_data()
         self.config.from_args(
-            ignore_errors=ignore_errors, omit=omit, include=include
+            ignore_errors=ignore_errors, omit=omit, include=include,
+            show_missing=show_missing,
             )
-        reporter = SummaryReporter(
-            self, show_missing, self.config.ignore_errors
-            )
-        reporter.report(morfs, outfile=file, config=self.config)
+        reporter = SummaryReporter(self, self.config)
+        return reporter.report(morfs, outfile=file)
 
     def annotate(self, morfs=None, directory=None, ignore_errors=None,
                     omit=None, include=None):
@@ -569,25 +630,39 @@ class coverage(object):
         See `coverage.report()` for other arguments.
 
         """
+        self._harvest_data()
         self.config.from_args(
             ignore_errors=ignore_errors, omit=omit, include=include
             )
-        reporter = AnnotateReporter(self, self.config.ignore_errors)
-        reporter.report(morfs, config=self.config, directory=directory)
+        reporter = AnnotateReporter(self, self.config)
+        reporter.report(morfs, directory=directory)
 
     def html_report(self, morfs=None, directory=None, ignore_errors=None,
-                    omit=None, include=None):
+                    omit=None, include=None, extra_css=None, title=None):
         """Generate an HTML report.
 
+        The HTML is written to `directory`.  The file "index.html" is the
+        overview starting point, with links to more detailed pages for
+        individual modules.
+
+        `extra_css` is a path to a file of other CSS to apply on the page.
+        It will be copied into the HTML directory.
+
+        `title` is a text string (not HTML) to use as the title of the HTML
+        report.
+
         See `coverage.report()` for other arguments.
 
+        Returns a float, the total percentage covered.
+
         """
+        self._harvest_data()
         self.config.from_args(
             ignore_errors=ignore_errors, omit=omit, include=include,
-            html_dir=directory,
+            html_dir=directory, extra_css=extra_css, html_title=title,
             )
-        reporter = HtmlReporter(self, self.config.ignore_errors)
-        reporter.report(morfs, config=self.config)
+        reporter = HtmlReporter(self, self.config)
+        return reporter.report(morfs)
 
     def xml_report(self, morfs=None, outfile=None, ignore_errors=None,
                     omit=None, include=None):
@@ -600,12 +675,16 @@ class coverage(object):
 
         See `coverage.report()` for other arguments.
 
+        Returns a float, the total percentage covered.
+
         """
+        self._harvest_data()
         self.config.from_args(
             ignore_errors=ignore_errors, omit=omit, include=include,
             xml_output=outfile,
             )
         file_to_close = None
+        delete_file = False
         if self.config.xml_output:
             if self.config.xml_output == '-':
                 outfile = sys.stdout
@@ -613,11 +692,17 @@ class coverage(object):
                 outfile = open(self.config.xml_output, "w")
                 file_to_close = outfile
         try:
-            reporter = XmlReporter(self, self.config.ignore_errors)
-            reporter.report(morfs, outfile=outfile, config=self.config)
+            try:
+                reporter = XmlReporter(self, self.config)
+                return reporter.report(morfs, outfile=outfile)
+            except CoverageException:
+                delete_file = True
+                raise
         finally:
             if file_to_close:
                 file_to_close.close()
+                if delete_file:
+                    file_be_gone(self.config.xml_output)
 
     def sysinfo(self):
         """Return a list of (key, value) pairs showing internal information."""
@@ -625,22 +710,43 @@ class coverage(object):
         import coverage as covmod
         import platform, re
 
+        try:
+            implementation = platform.python_implementation()
+        except AttributeError:
+            implementation = "unknown"
+
         info = [
             ('version', covmod.__version__),
             ('coverage', covmod.__file__),
             ('cover_dir', self.cover_dir),
             ('pylib_dirs', self.pylib_dirs),
             ('tracer', self.collector.tracer_name()),
+            ('config_files', self.config.attempted_config_files),
+            ('configs_read', self.config.config_files),
             ('data_path', self.data.filename),
             ('python', sys.version.replace('\n', '')),
             ('platform', platform.platform()),
+            ('implementation', implementation),
+            ('executable', sys.executable),
             ('cwd', os.getcwd()),
             ('path', sys.path),
-            ('environment', [
-                ("%s = %s" % (k, v)) for k, v in os.environ.items()
-                    if re.search("^COV|^PY", k)
-                ]),
+            ('environment', sorted([
+                ("%s = %s" % (k, v)) for k, v in iitems(os.environ)
+                    if re.search(r"^COV|^PY", k)
+                ])),
+            ('command_line', " ".join(getattr(sys, 'argv', ['???']))),
             ]
+        if self.source_match:
+            info.append(('source_match', self.source_match.info()))
+        if self.include_match:
+            info.append(('include_match', self.include_match.info()))
+        if self.omit_match:
+            info.append(('omit_match', self.omit_match.info()))
+        if self.cover_match:
+            info.append(('cover_match', self.cover_match.info()))
+        if self.pylib_match:
+            info.append(('pylib_match', self.pylib_match.info()))
+
         return info
 
 
@@ -667,7 +773,10 @@ def process_startup():
     cps = os.environ.get("COVERAGE_PROCESS_START")
     if cps:
         cov = coverage(config_file=cps, auto_data=True)
-        if os.environ.get("COVERAGE_COVERAGE"):
-            # Measuring coverage within coverage.py takes yet more trickery.
-            cov.cover_dir = "Please measure coverage.py!"
         cov.start()
+        cov._warn_no_data = False
+        cov._warn_unimported_source = False
+
+
+# A hack for debugging testing in subprocesses.
+_TEST_NAME_FILE = "" #"/tmp/covtest.txt"

python/helpers/coverage/data.py

@@ -2,7 +2,9 @@
 
 import os
 
-from coverage.backward import pickle, sorted        # pylint: disable=W0622
+from coverage.backward import iitems, pickle, sorted    # pylint: disable=W0622
+from coverage.files import PathAliases
+from coverage.misc import file_be_gone
 
 
 class CoverageData(object):
@@ -21,15 +23,18 @@ class CoverageData(object):
 
     """
 
-    def __init__(self, basename=None, collector=None):
+    def __init__(self, basename=None, collector=None, debug=None):
         """Create a CoverageData.
 
         `basename` is the name of the file to use for storing data.
 
         `collector` is a string describing the coverage measurement software.
 
+        `debug` is a `DebugControl` object for writing debug messages.
+
         """
         self.collector = collector or 'unknown'
+        self.debug = debug
 
         self.use_file = True
 
@@ -59,10 +64,6 @@ class CoverageData(object):
         #
         self.arcs = {}
 
-        self.os = os
-        self.sorted = sorted
-        self.pickle = pickle
-
     def usefile(self, use_file=True):
         """Set whether or not to use a disk file for data."""
         self.use_file = use_file
@@ -92,21 +93,21 @@ class CoverageData(object):
     def erase(self):
         """Erase the data, both in this object, and from its file storage."""
         if self.use_file:
-            if self.filename and os.path.exists(self.filename):
-                os.remove(self.filename)
+            if self.filename:
+                file_be_gone(self.filename)
         self.lines = {}
         self.arcs = {}
 
     def line_data(self):
         """Return the map from filenames to lists of line numbers executed."""
         return dict(
-            [(f, self.sorted(lmap.keys())) for f, lmap in self.lines.items()]
+            [(f, sorted(lmap.keys())) for f, lmap in iitems(self.lines)]
             )
 
     def arc_data(self):
         """Return the map from filenames to lists of line number pairs."""
         return dict(
-            [(f, self.sorted(amap.keys())) for f, amap in self.arcs.items()]
+            [(f, sorted(amap.keys())) for f, amap in iitems(self.arcs)]
             )
 
     def write_file(self, filename):
@@ -123,10 +124,13 @@ class CoverageData(object):
         if self.collector:
             data['collector'] = self.collector
 
+        if self.debug and self.debug.should('dataio'):
+            self.debug.write("Writing data to %r" % (filename,))
+
         # Write the pickle to the file.
         fdata = open(filename, 'wb')
         try:
-            self.pickle.dump(data, fdata, 2)
+            pickle.dump(data, fdata, 2)
         finally:
             fdata.close()
 
@@ -136,6 +140,8 @@ class CoverageData(object):
 
     def raw_data(self, filename):
         """Return the raw pickled data from `filename`."""
+        if self.debug and self.debug.should('dataio'):
+            self.debug.write("Reading data from %r" % (filename,))
         fdata = open(filename, 'rb')
         try:
             data = pickle.load(fdata)
@@ -158,33 +164,39 @@ class CoverageData(object):
                 # Unpack the 'lines' item.
                 lines = dict([
                     (f, dict.fromkeys(linenos, None))
-                        for f, linenos in data.get('lines', {}).items()
+                        for f, linenos in iitems(data.get('lines', {}))
                     ])
                 # Unpack the 'arcs' item.
                 arcs = dict([
                     (f, dict.fromkeys(arcpairs, None))
-                        for f, arcpairs in data.get('arcs', {}).items()
+                        for f, arcpairs in iitems(data.get('arcs', {}))
                     ])
         except Exception:
             pass
         return lines, arcs
 
-    def combine_parallel_data(self):
+    def combine_parallel_data(self, aliases=None):
         """Combine a number of data files together.
 
         Treat `self.filename` as a file prefix, and combine the data from all
         of the data files starting with that prefix plus a dot.
 
+        If `aliases` is provided, it's a `PathAliases` object that is used to
+        re-map paths to match the local machine's.
+
         """
+        aliases = aliases or PathAliases()
         data_dir, local = os.path.split(self.filename)
         localdot = local + '.'
         for f in os.listdir(data_dir or '.'):
             if f.startswith(localdot):
                 full_path = os.path.join(data_dir, f)
                 new_lines, new_arcs = self._read_file(full_path)
-                for filename, file_data in new_lines.items():
+                for filename, file_data in iitems(new_lines):
+                    filename = aliases.map(filename)
                     self.lines.setdefault(filename, {}).update(file_data)
-                for filename, file_data in new_arcs.items():
+                for filename, file_data in iitems(new_arcs):
+                    filename = aliases.map(filename)
                     self.arcs.setdefault(filename, {}).update(file_data)
                 if f != local:
                     os.remove(full_path)
@@ -195,7 +207,7 @@ class CoverageData(object):
         `line_data` is { filename: { lineno: None, ... }, ...}
 
         """
-        for filename, linenos in line_data.items():
+        for filename, linenos in iitems(line_data):
             self.lines.setdefault(filename, {}).update(linenos)
 
     def add_arc_data(self, arc_data):
@@ -204,7 +216,7 @@ class CoverageData(object):
         `arc_data` is { filename: { (l1,l2): None, ... }, ...}
 
         """
-        for filename, arcs in arc_data.items():
+        for filename, arcs in iitems(arc_data):
             self.arcs.setdefault(filename, {}).update(arcs)
 
     def touch_file(self, filename):
@@ -245,8 +257,8 @@ class CoverageData(object):
         if fullpath:
             filename_fn = lambda f: f
         else:
-            filename_fn = self.os.path.basename
-        for filename, lines in self.lines.items():
+            filename_fn = os.path.basename
+        for filename, lines in iitems(self.lines):
             summ[filename_fn(filename)] = len(lines)
         return summ
 

python/helpers/coverage/debug.py

@@ -0,0 +1,54 @@
+"""Control of and utilities for debugging."""
+
+import os
+
+
+# When debugging, it can be helpful to force some options, especially when
+# debugging the configuration mechanisms you usually use to control debugging!
+# This is a list of forced debugging options.
+FORCED_DEBUG = []
+
+
+class DebugControl(object):
+    """Control and output for debugging."""
+
+    def __init__(self, options, output):
+        """Configure the options and output file for debugging."""
+        self.options = options
+        self.output = output
+
+    def should(self, option):
+        """Decide whether to output debug information in category `option`."""
+        return (option in self.options or option in FORCED_DEBUG)
+
+    def write(self, msg):
+        """Write a line of debug output."""
+        if self.should('pid'):
+            msg = "pid %5d: %s" % (os.getpid(), msg)
+        self.output.write(msg+"\n")
+        self.output.flush()
+
+    def write_formatted_info(self, info):
+        """Write a sequence of (label,data) pairs nicely."""
+        for line in info_formatter(info):
+            self.write(" %s" % line)
+
+
+def info_formatter(info):
+    """Produce a sequence of formatted lines from info.
+
+    `info` is a sequence of pairs (label, data).  The produced lines are
+    nicely formatted, ready to print.
+
+    """
+    label_len = max([len(l) for l, _d in info])
+    for label, data in info:
+        if data == []:
+            data = "-none-"
+        if isinstance(data, (list, tuple)):
+            prefix = "%*s:" % (label_len, label)
+            for e in data:
+                yield "%*s %s" % (label_len+1, prefix, e)
+                prefix = ""
+        else:
+            yield "%*s: %s" % (label_len, label, data)

python/helpers/coverage/execfile.py

@@ -1,9 +1,11 @@
 """Execute files of Python code."""
 
-import imp, os, sys
-
+import imp
+import marshal
+import os
+import sys
 from coverage.backward import exec_code_object, open_source
-from coverage.misc import NoSource, ExceptionDuringRun
+from coverage.misc import ExceptionDuringRun, NoCode, NoSource
 
 
 try:
@@ -65,6 +67,8 @@ def run_python_module(modulename, args):
             openfile.close()
 
     # Finally, hand the file off to run_python_file for execution.
+    pathname = os.path.abspath(pathname)
+    args[0] = pathname
     run_python_file(pathname, args, package=packagename)
 
 
@@ -82,34 +86,22 @@ def run_python_file(filename, args, package=None):
     main_mod = imp.new_module('__main__')
     sys.modules['__main__'] = main_mod
     main_mod.__file__ = filename
-    main_mod.__package__ = package
+    if package:
+        main_mod.__package__ = package
     main_mod.__builtins__ = BUILTINS
 
-    # Set sys.argv and the first path element properly.
+    # Set sys.argv properly.
     old_argv = sys.argv
-    old_path0 = sys.path[0]
     sys.argv = args
-    sys.path[0] = os.path.abspath(os.path.dirname(filename))
 
     try:
-        # Open the source file.
-        try:
-            source_file = open_source(filename)
-        except IOError:
-            raise NoSource("No file to run: %r" % filename)
+        # Make a code object somehow.
+        if filename.endswith(".pyc") or filename.endswith(".pyo"):
+            code = make_code_from_pyc(filename)
+        else:
+            code = make_code_from_py(filename)
 
-        try:
-            source = source_file.read()
-        finally:
-            source_file.close()
-
-        # We have the source.  `compile` still needs the last line to be clean,
-        # so make sure it is, then compile a code object from it.
-        if source[-1] != '\n':
-            source += '\n'
-        code = compile(source, filename, "exec")
-
-        # Execute the source file.
+        # Execute the code object.
         try:
             exec_code_object(code, main_mod.__dict__)
         except SystemExit:
@@ -130,4 +122,52 @@ def run_python_file(filename, args, package=None):
 
         # Restore the old argv and path
         sys.argv = old_argv
-        sys.path[0] = old_path0
+
+def make_code_from_py(filename):
+    """Get source from `filename` and make a code object of it."""
+    # Open the source file.
+    try:
+        source_file = open_source(filename)
+    except IOError:
+        raise NoSource("No file to run: %r" % filename)
+
+    try:
+        source = source_file.read()
+    finally:
+        source_file.close()
+
+    # We have the source.  `compile` still needs the last line to be clean,
+    # so make sure it is, then compile a code object from it.
+    if not source or source[-1] != '\n':
+        source += '\n'
+    code = compile(source, filename, "exec")
+
+    return code
+
+
+def make_code_from_pyc(filename):
+    """Get a code object from a .pyc file."""
+    try:
+        fpyc = open(filename, "rb")
+    except IOError:
+        raise NoCode("No file to run: %r" % filename)
+
+    try:
+        # First four bytes are a version-specific magic number.  It has to
+        # match or we won't run the file.
+        magic = fpyc.read(4)
+        if magic != imp.get_magic():
+            raise NoCode("Bad magic number in .pyc file")
+
+        # Skip the junk in the header that we don't need.
+        fpyc.read(4)            # Skip the moddate.
+        if sys.version_info >= (3, 3):
+            # 3.3 added another long to the header (size), skip it.
+            fpyc.read(4)
+
+        # The rest of the file is the code object we want.
+        code = marshal.load(fpyc)
+    finally:
+        fpyc.close()
+
+    return code

python/helpers/coverage/files.py

@@ -1,23 +1,27 @@
 """File wrangling."""
 
 from coverage.backward import to_string
-import fnmatch, os, sys
+from coverage.misc import CoverageException
+import fnmatch
+import os
+import os.path
+import re
+import sys
+import ntpath
+import posixpath
+
 
 class FileLocator(object):
     """Understand how filenames work."""
 
     def __init__(self):
         # The absolute path to our current directory.
-        self.relative_dir = self.abs_file(os.curdir) + os.sep
+        self.relative_dir = os.path.normcase(abs_file(os.curdir) + os.sep)
 
         # Cache of results of calling the canonical_filename() method, to
         # avoid duplicating work.
         self.canonical_filename_cache = {}
 
-    def abs_file(self, filename):
-        """Return the absolute normalized form of `filename`."""
-        return os.path.normcase(os.path.abspath(os.path.realpath(filename)))
-
     def relative_filename(self, filename):
         """Return the relative form of `filename`.
 
@@ -25,8 +29,9 @@ class FileLocator(object):
         `FileLocator` was constructed.
 
         """
-        if filename.startswith(self.relative_dir):
-            filename = filename.replace(self.relative_dir, "")
+        fnorm = os.path.normcase(filename)
+        if fnorm.startswith(self.relative_dir):
+            filename = filename[len(self.relative_dir):]
         return filename
 
     def canonical_filename(self, filename):
@@ -36,19 +41,15 @@ class FileLocator(object):
 
         """
         if filename not in self.canonical_filename_cache:
-            f = filename
-            if os.path.isabs(f) and not os.path.exists(f):
-                if self.get_zip_data(f) is None:
-                    f = os.path.basename(f)
-            if not os.path.isabs(f):
+            if not os.path.isabs(filename):
                 for path in [os.curdir] + sys.path:
                     if path is None:
                         continue
-                    g = os.path.join(path, f)
-                    if os.path.exists(g):
-                        f = g
+                    f = os.path.join(path, filename)
+                    if os.path.exists(f):
+                        filename = f
                         break
-            cf = self.abs_file(f)
+            cf = abs_file(filename)
             self.canonical_filename_cache[filename] = cf
         return self.canonical_filename_cache[filename]
 
@@ -77,6 +78,78 @@ class FileLocator(object):
         return None
 
 
+if sys.platform == 'win32':
+
+    def actual_path(path):
+        """Get the actual path of `path`, including the correct case."""
+        if path in actual_path.cache:
+            return actual_path.cache[path]
+
+        head, tail = os.path.split(path)
+        if not tail:
+            actpath = head
+        elif not head:
+            actpath = tail
+        else:
+            head = actual_path(head)
+            if head in actual_path.list_cache:
+                files = actual_path.list_cache[head]
+            else:
+                try:
+                    files = os.listdir(head)
+                except OSError:
+                    files = []
+                actual_path.list_cache[head] = files
+            normtail = os.path.normcase(tail)
+            for f in files:
+                if os.path.normcase(f) == normtail:
+                    tail = f
+                    break
+            actpath = os.path.join(head, tail)
+        actual_path.cache[path] = actpath
+        return actpath
+
+    actual_path.cache = {}
+    actual_path.list_cache = {}
+
+else:
+    def actual_path(filename):
+        """The actual path for non-Windows platforms."""
+        return filename
+
+
+def abs_file(filename):
+    """Return the absolute normalized form of `filename`."""
+    path = os.path.expandvars(os.path.expanduser(filename))
+    path = os.path.abspath(os.path.realpath(path))
+    path = actual_path(path)
+    return path
+
+
+def isabs_anywhere(filename):
+    """Is `filename` an absolute path on any OS?"""
+    return ntpath.isabs(filename) or posixpath.isabs(filename)
+
+
+def prep_patterns(patterns):
+    """Prepare the file patterns for use in a `FnmatchMatcher`.
+
+    If a pattern starts with a wildcard, it is used as a pattern
+    as-is.  If it does not start with a wildcard, then it is made
+    absolute with the current directory.
+
+    If `patterns` is None, an empty list is returned.
+
+    """
+    prepped = []
+    for p in patterns or []:
+        if p.startswith("*") or p.startswith("?"):
+            prepped.append(p)
+        else:
+            prepped.append(abs_file(p))
+    return prepped
+
+
 class TreeMatcher(object):
     """A matcher for files in a tree."""
     def __init__(self, directories):
@@ -85,6 +158,10 @@ class TreeMatcher(object):
     def __repr__(self):
         return "<TreeMatcher %r>" % self.dirs
 
+    def info(self):
+        """A list of strings for displaying when dumping state."""
+        return self.dirs
+
     def add(self, directory):
         """Add another directory to the list we match for."""
         self.dirs.append(directory)
@@ -110,6 +187,10 @@ class FnmatchMatcher(object):
     def __repr__(self):
         return "<FnmatchMatcher %r>" % self.pats
 
+    def info(self):
+        """A list of strings for displaying when dumping state."""
+        return self.pats
+
     def match(self, fpath):
         """Does `fpath` match one of our filename patterns?"""
         for pat in self.pats:
@@ -118,14 +199,117 @@ class FnmatchMatcher(object):
         return False
 
 
+def sep(s):
+    """Find the path separator used in this string, or os.sep if none."""
+    sep_match = re.search(r"[\\/]", s)
+    if sep_match:
+        the_sep = sep_match.group(0)
+    else:
+        the_sep = os.sep
+    return the_sep
+
+
+class PathAliases(object):
+    """A collection of aliases for paths.
+
+    When combining data files from remote machines, often the paths to source
+    code are different, for example, due to OS differences, or because of
+    serialized checkouts on continuous integration machines.
+
+    A `PathAliases` object tracks a list of pattern/result pairs, and can
+    map a path through those aliases to produce a unified path.
+
+    `locator` is a FileLocator that is used to canonicalize the results.
+
+    """
+    def __init__(self, locator=None):
+        self.aliases = []
+        self.locator = locator
+
+    def add(self, pattern, result):
+        """Add the `pattern`/`result` pair to the list of aliases.
+
+        `pattern` is an `fnmatch`-style pattern.  `result` is a simple
+        string.  When mapping paths, if a path starts with a match against
+        `pattern`, then that match is replaced with `result`.  This models
+        isomorphic source trees being rooted at different places on two
+        different machines.
+
+        `pattern` can't end with a wildcard component, since that would
+        match an entire tree, and not just its root.
+
+        """
+        # The pattern can't end with a wildcard component.
+        pattern = pattern.rstrip(r"\/")
+        if pattern.endswith("*"):
+            raise CoverageException("Pattern must not end with wildcards.")
+        pattern_sep = sep(pattern)
+
+        # The pattern is meant to match a filepath.  Let's make it absolute
+        # unless it already is, or is meant to match any prefix.
+        if not pattern.startswith('*') and not isabs_anywhere(pattern):
+            pattern = abs_file(pattern)
+        pattern += pattern_sep
+
+        # Make a regex from the pattern.  fnmatch always adds a \Z or $ to
+        # match the whole string, which we don't want.
+        regex_pat = fnmatch.translate(pattern).replace(r'\Z(', '(')
+        if regex_pat.endswith("$"):
+            regex_pat = regex_pat[:-1]
+        # We want */a/b.py to match on Windows too, so change slash to match
+        # either separator.
+        regex_pat = regex_pat.replace(r"\/", r"[\\/]")
+        # We want case-insensitive matching, so add that flag.
+        regex = re.compile(r"(?i)" + regex_pat)
+
+        # Normalize the result: it must end with a path separator.
+        result_sep = sep(result)
+        result = result.rstrip(r"\/") + result_sep
+        self.aliases.append((regex, result, pattern_sep, result_sep))
+
+    def map(self, path):
+        """Map `path` through the aliases.
+
+        `path` is checked against all of the patterns.  The first pattern to
+        match is used to replace the root of the path with the result root.
+        Only one pattern is ever used.  If no patterns match, `path` is
+        returned unchanged.
+
+        The separator style in the result is made to match that of the result
+        in the alias.
+
+        """
+        for regex, result, pattern_sep, result_sep in self.aliases:
+            m = regex.match(path)
+            if m:
+                new = path.replace(m.group(0), result)
+                if pattern_sep != result_sep:
+                    new = new.replace(pattern_sep, result_sep)
+                if self.locator:
+                    new = self.locator.canonical_filename(new)
+                return new
+        return path
+
+
 def find_python_files(dirname):
-    """Yield all of the importable Python files in `dirname`, recursively."""
-    for dirpath, dirnames, filenames in os.walk(dirname, topdown=True):
-        if '__init__.py' not in filenames:
+    """Yield all of the importable Python files in `dirname`, recursively.
+
+    To be importable, the files have to be in a directory with a __init__.py,
+    except for `dirname` itself, which isn't required to have one.  The
+    assumption is that `dirname` was specified directly, so the user knows
+    best, but subdirectories are checked for a __init__.py to be sure we only
+    find the importable files.
+
+    """
+    for i, (dirpath, dirnames, filenames) in enumerate(os.walk(dirname)):
+        if i > 0 and '__init__.py' not in filenames:
             # If a directory doesn't have __init__.py, then it isn't
             # importable and neither are its files
             del dirnames[:]
             continue
         for filename in filenames:
-            if fnmatch.fnmatch(filename, "*.py"):
+            # We're only interested in files that look like reasonable Python
+            # files: Must end with .py or .pyw, and must not have certain funny
+            # characters that probably mean they are editor junk.
+            if re.match(r"^[^.#~!$@%^&*()+=,]+\.pyw?$", filename):
                 yield os.path.join(dirpath, filename)

python/helpers/coverage/fullcoverage/encodings.py

@@ -0,0 +1,56 @@
+"""Imposter encodings module that installs a coverage-style tracer.
+
+This is NOT the encodings module; it is an imposter that sets up tracing
+instrumentation and then replaces itself with the real encodings module.
+
+If the directory that holds this file is placed first in the PYTHONPATH when
+using "coverage" to run Python's tests, then this file will become the very
+first module imported by the internals of Python 3.  It installs a
+coverage-compatible trace function that can watch Standard Library modules
+execute from the very earliest stages of Python's own boot process.  This fixes
+a problem with coverage - that it starts too late to trace the coverage of many
+of the most fundamental modules in the Standard Library.
+
+"""
+
+import sys
+
+class FullCoverageTracer(object):
+    def __init__(self):
+        # `traces` is a list of trace events.  Frames are tricky: the same
+        # frame object is used for a whole scope, with new line numbers
+        # written into it.  So in one scope, all the frame objects are the
+        # same object, and will eventually all will point to the last line
+        # executed.  So we keep the line numbers alongside the frames.
+        # The list looks like:
+        #
+        #   traces = [
+        #       ((frame, event, arg), lineno), ...
+        #       ]
+        #
+        self.traces = []
+
+    def fullcoverage_trace(self, *args):
+        frame, event, arg = args
+        self.traces.append((args, frame.f_lineno))
+        return self.fullcoverage_trace
+
+sys.settrace(FullCoverageTracer().fullcoverage_trace)
+
+# In coverage/files.py is actual_filename(), which uses glob.glob.  I don't
+# understand why, but that use of glob borks everything if fullcoverage is in
+# effect.  So here we make an ugly hail-mary pass to switch off glob.glob over
+# there.  This means when using fullcoverage, Windows path names will not be
+# their actual case.
+
+#sys.fullcoverage = True
+
+# Finally, remove our own directory from sys.path; remove ourselves from
+# sys.modules; and re-import "encodings", which will be the real package
+# this time.  Note that the delete from sys.modules dictionary has to
+# happen last, since all of the symbols in this module will become None
+# at that exact moment, including "sys".
+
+parentdir = max(filter(__file__.startswith, sys.path), key=len)
+sys.path.remove(parentdir)
+del sys.modules['encodings']

python/helpers/coverage/html.py

@@ -1,21 +1,47 @@
 """HTML reporting for Coverage."""
 
-import os, re, shutil
-
+import os
+import re
+import shutil
+import sys
 import coverage
 from coverage.backward import pickle
 from coverage.misc import CoverageException, Hasher
-from coverage.phystokens import source_token_lines
+from coverage.phystokens import source_token_lines, source_encoding
 from coverage.report import Reporter
+from coverage.results import Numbers
 from coverage.templite import Templite
 
-# Disable pylint msg W0612, because a bunch of variables look unused, but
-# they're accessed in a Templite context via locals().
-# pylint: disable=W0612
 
-def data_filename(fname):
-    """Return the path to a data file of ours."""
-    return os.path.join(os.path.split(__file__)[0], fname)
+# Static files are looked for in a list of places.
+STATIC_PATH = [
+    # The place Debian puts system Javascript libraries.
+    "/usr/share/javascript",
+
+    # Our htmlfiles directory.
+    os.path.join(os.path.dirname(__file__), "htmlfiles"),
+]
+
+def data_filename(fname, pkgdir=""):
+    """Return the path to a data file of ours.
+
+    The file is searched for on `STATIC_PATH`, and the first place it's found,
+    is returned.
+
+    Each directory in `STATIC_PATH` is searched as-is, and also, if `pkgdir`
+    is provided, at that subdirectory.
+
+    """
+    for static_dir in STATIC_PATH:
+        static_filename = os.path.join(static_dir, fname)
+        if os.path.exists(static_filename):
+            return static_filename
+        if pkgdir:
+            static_filename = os.path.join(static_dir, pkgdir, fname)
+            if os.path.exists(static_filename):
+                return static_filename
+    raise CoverageException("Couldn't find static file %r" % fname)
+
 
 def data(fname):
     """Return the contents of a data file of ours."""
@@ -31,26 +57,27 @@ class HtmlReporter(Reporter):
 
     # These files will be copied from the htmlfiles dir to the output dir.
     STATIC_FILES = [
-            "style.css",
-            "jquery-1.4.3.min.js",
-            "jquery.hotkeys.js",
-            "jquery.isonscreen.js",
-            "jquery.tablesorter.min.js",
-            "coverage_html.js",
-            "keybd_closed.png",
-            "keybd_open.png",
+            ("style.css", ""),
+            ("jquery.min.js", "jquery"),
+            ("jquery.hotkeys.js", "jquery-hotkeys"),
+            ("jquery.isonscreen.js", "jquery-isonscreen"),
+            ("jquery.tablesorter.min.js", "jquery-tablesorter"),
+            ("coverage_html.js", ""),
+            ("keybd_closed.png", ""),
+            ("keybd_open.png", ""),
             ]
 
-    def __init__(self, cov, ignore_errors=False):
-        super(HtmlReporter, self).__init__(cov, ignore_errors)
+    def __init__(self, cov, config):
+        super(HtmlReporter, self).__init__(cov, config)
         self.directory = None
         self.template_globals = {
             'escape': escape,
+            'title': self.config.html_title,
             '__url__': coverage.__url__,
             '__version__': coverage.__version__,
             }
         self.source_tmpl = Templite(
-            data("htmlfiles/pyfile.html"), self.template_globals
+            data("pyfile.html"), self.template_globals
             )
 
         self.coverage = cov
@@ -58,29 +85,34 @@ class HtmlReporter(Reporter):
         self.files = []
         self.arcs = self.coverage.data.has_arcs()
         self.status = HtmlStatus()
+        self.extra_css = None
+        self.totals = Numbers()
 
-    def report(self, morfs, config=None):
+    def report(self, morfs):
         """Generate an HTML report for `morfs`.
 
-        `morfs` is a list of modules or filenames.  `config` is a
-        CoverageConfig instance.
+        `morfs` is a list of modules or filenames.
 
         """
-        assert config.html_dir, "must provide a directory for html reporting"
+        assert self.config.html_dir, "must give a directory for html reporting"
 
         # Read the status data.
-        self.status.read(config.html_dir)
+        self.status.read(self.config.html_dir)
 
         # Check that this run used the same settings as the last run.
         m = Hasher()
-        m.update(config)
+        m.update(self.config)
         these_settings = m.digest()
         if self.status.settings_hash() != these_settings:
             self.status.reset()
             self.status.set_settings_hash(these_settings)
 
+        # The user may have extra CSS they want copied.
+        if self.config.extra_css:
+            self.extra_css = os.path.basename(self.config.extra_css)
+
         # Process all the files.
-        self.report_files(self.html_file, morfs, config, config.html_dir)
+        self.report_files(self.html_file, morfs, self.config.html_dir)
 
         if not self.files:
             raise CoverageException("No data to report.")
@@ -88,13 +120,34 @@ class HtmlReporter(Reporter):
         # Write the index file.
         self.index_file()
 
-        # Create the once-per-directory files.
-        for static in self.STATIC_FILES:
+        self.make_local_static_report_files()
+
+        return self.totals.pc_covered
+
+    def make_local_static_report_files(self):
+        """Make local instances of static files for HTML report."""
+        # The files we provide must always be copied.
+        for static, pkgdir in self.STATIC_FILES:
             shutil.copyfile(
-                data_filename("htmlfiles/" + static),
+                data_filename(static, pkgdir),
                 os.path.join(self.directory, static)
                 )
 
+        # The user may have extra CSS they want copied.
+        if self.extra_css:
+            shutil.copyfile(
+                self.config.extra_css,
+                os.path.join(self.directory, self.extra_css)
+                )
+
+    def write_html(self, fname, html):
+        """Write `html` to `fname`, properly encoded."""
+        fout = open(fname, "wb")
+        try:
+            fout.write(html.encode('ascii', 'xmlcharrefreplace'))
+        finally:
+            fout.close()
+
     def file_hash(self, source, cu):
         """Compute a hash that changes if the file needs to be re-reported."""
         m = Hasher()
@@ -121,11 +174,20 @@ class HtmlReporter(Reporter):
 
         self.status.set_file_hash(flat_rootname, this_hash)
 
+        # If need be, determine the encoding of the source file. We use it
+        # later to properly write the HTML.
+        if sys.version_info < (3, 0):
+            encoding = source_encoding(source)
+            # Some UTF8 files have the dreaded UTF8 BOM. If so, junk it.
+            if encoding.startswith("utf-8") and source[:3] == "\xef\xbb\xbf":
+                source = source[3:]
+                encoding = "utf-8"
+
+        # Get the numbers for this file.
         nums = analysis.numbers
 
-        missing_branch_arcs = analysis.missing_branch_arcs()
-        n_par = 0   # accumulated below.
-        arcs = self.arcs
+        if self.arcs:
+            missing_branch_arcs = analysis.missing_branch_arcs()
 
         # These classes determine which lines are highlighted by default.
         c_run = "run hide_run"
@@ -149,7 +211,6 @@ class HtmlReporter(Reporter):
                 line_class.append(c_mis)
             elif self.arcs and lineno in missing_branch_arcs:
                 line_class.append(c_par)
-                n_par += 1
                 annlines = []
                 for b in missing_branch_arcs[lineno]:
                     if b < 0:
@@ -184,19 +245,22 @@ class HtmlReporter(Reporter):
             })
 
         # Write the HTML page for this file.
+        html = spaceless(self.source_tmpl.render({
+            'c_exc': c_exc, 'c_mis': c_mis, 'c_par': c_par, 'c_run': c_run,
+            'arcs': self.arcs, 'extra_css': self.extra_css,
+            'cu': cu, 'nums': nums, 'lines': lines,
+        }))
+
+        if sys.version_info < (3, 0):
+            html = html.decode(encoding)
+
         html_filename = flat_rootname + ".html"
         html_path = os.path.join(self.directory, html_filename)
-        html = spaceless(self.source_tmpl.render(locals()))
-        fhtml = open(html_path, 'w')
-        try:
-            fhtml.write(html)
-        finally:
-            fhtml.close()
+        self.write_html(html_path, html)
 
         # Save this file's information for the index file.
         index_info = {
             'nums': nums,
-            'par': n_par,
             'html_filename': html_filename,
             'name': cu.name,
             }
@@ -206,19 +270,24 @@ class HtmlReporter(Reporter):
     def index_file(self):
         """Write the index.html file for this report."""
         index_tmpl = Templite(
-            data("htmlfiles/index.html"), self.template_globals
+            data("index.html"), self.template_globals
             )
 
-        files = self.files
-        arcs = self.arcs
+        self.totals = sum([f['nums'] for f in self.files])
 
-        totals = sum([f['nums'] for f in files])
+        html = index_tmpl.render({
+            'arcs': self.arcs,
+            'extra_css': self.extra_css,
+            'files': self.files,
+            'totals': self.totals,
+        })
 
-        fhtml = open(os.path.join(self.directory, "index.html"), "w")
-        try:
-            fhtml.write(index_tmpl.render(locals()))
-        finally:
-            fhtml.close()
+        if sys.version_info < (3, 0):
+            html = html.decode("utf-8")
+        self.write_html(
+            os.path.join(self.directory, "index.html"),
+            html
+            )
 
         # Write the latest hashes for next time.
         self.status.write(self.directory)
@@ -243,8 +312,12 @@ class HtmlStatus(object):
         usable = False
         try:
             status_file = os.path.join(directory, self.STATUS_FILE)
-            status = pickle.load(open(status_file, "rb"))
-        except IOError:
+            fstatus = open(status_file, "rb")
+            try:
+                status = pickle.load(fstatus)
+            finally:
+                fstatus.close()
+        except (IOError, ValueError):
             usable = False
         else:
             usable = True
@@ -321,5 +394,5 @@ def spaceless(html):
     Get rid of some.
 
     """
-    html = re.sub(">\s+<p ", ">\n<p ", html)
+    html = re.sub(r">\s+<p ", ">\n<p ", html)
     return html

python/helpers/coverage/htmlfiles/coverage_html.js

@@ -122,6 +122,11 @@ coverage.pyfile_ready = function ($) {
         .bind('keydown', '1', coverage.to_first_chunk)
         ;
 
+    $(".button_toggle_run").click(function (evt) {coverage.toggle_lines(evt.target, "run");});
+    $(".button_toggle_exc").click(function (evt) {coverage.toggle_lines(evt.target, "exc");});
+    $(".button_toggle_mis").click(function (evt) {coverage.toggle_lines(evt.target, "mis");});
+    $(".button_toggle_par").click(function (evt) {coverage.toggle_lines(evt.target, "par");});
+
     coverage.assign_shortkeys();
     coverage.wire_up_help_panel();
 };
@@ -369,4 +374,3 @@ coverage.scroll_window = function (to_pos) {
 coverage.finish_scrolling = function () {
     $("html,body").stop(true, true);
 };
-

python/helpers/coverage/htmlfiles/index.html

@@ -2,9 +2,12 @@
 <html>
 <head>
     <meta http-equiv='Content-Type' content='text/html; charset=utf-8'>
-    <title>Coverage report</title>
+    <title>{{ title|escape }}</title>
     <link rel='stylesheet' href='style.css' type='text/css'>
-    <script type='text/javascript' src='jquery-1.4.3.min.js'></script>
+    {% if extra_css %}
+        <link rel='stylesheet' href='{{ extra_css }}' type='text/css'>
+    {% endif %}
+    <script type='text/javascript' src='jquery.min.js'></script>
     <script type='text/javascript' src='jquery.tablesorter.min.js'></script>
     <script type='text/javascript' src='jquery.hotkeys.js'></script>
     <script type='text/javascript' src='coverage_html.js'></script>
@@ -16,7 +19,7 @@
 
 <div id='header'>
     <div class='content'>
-        <h1>Coverage report:
+        <h1>{{ title|escape }}:
             <span class='pc_cov'>{{totals.pc_covered_str}}%</span>
         </h1>
         <img id='keyboard_icon' src='keybd_closed.png'>
@@ -66,7 +69,7 @@
                 <td>{{totals.n_excluded}}</td>
                 {% if arcs %}
                 <td>{{totals.n_branches}}</td>
-                <td>{{totals.n_missing_branches}}</td>
+                <td>{{totals.n_partial_branches}}</td>
                 {% endif %}
                 <td class='right'>{{totals.pc_covered_str}}%</td>
             </tr>
@@ -80,7 +83,7 @@
                 <td>{{file.nums.n_excluded}}</td>
                 {% if arcs %}
                 <td>{{file.nums.n_branches}}</td>
-                <td>{{file.nums.n_missing_branches}}</td>
+                <td>{{file.nums.n_partial_branches}}</td>
                 {% endif %}
                 <td class='right'>{{file.nums.pc_covered_str}}%</td>
             </tr>

python/helpers/coverage/htmlfiles/pyfile.html

@@ -7,7 +7,10 @@
     <meta http-equiv='X-UA-Compatible' content='IE=emulateIE7' />
     <title>Coverage for {{cu.name|escape}}: {{nums.pc_covered_str}}%</title>
     <link rel='stylesheet' href='style.css' type='text/css'>
-    <script type='text/javascript' src='jquery-1.4.3.min.js'></script>
+    {% if extra_css %}
+        <link rel='stylesheet' href='{{ extra_css }}' type='text/css'>
+    {% endif %}
+    <script type='text/javascript' src='jquery.min.js'></script>
     <script type='text/javascript' src='jquery.hotkeys.js'></script>
     <script type='text/javascript' src='jquery.isonscreen.js'></script>
     <script type='text/javascript' src='coverage_html.js'></script>
@@ -24,12 +27,12 @@
         </h1>
         <img id='keyboard_icon' src='keybd_closed.png'>
         <h2 class='stats'>
-            {{nums.n_statements}} statements
-            <span class='{{c_run}} shortkey_r' onclick='coverage.toggle_lines(this, "run")'>{{nums.n_executed}} run</span>
-            <span class='{{c_mis}} shortkey_m' onclick='coverage.toggle_lines(this, "mis")'>{{nums.n_missing}} missing</span>
-            <span class='{{c_exc}} shortkey_x' onclick='coverage.toggle_lines(this, "exc")'>{{nums.n_excluded}} excluded</span>
+            {{nums.n_statements}} statements &nbsp;
+            <span class='{{c_run}} shortkey_r button_toggle_run'>{{nums.n_executed}} run</span>
+            <span class='{{c_mis}} shortkey_m button_toggle_mis'>{{nums.n_missing}} missing</span>
+            <span class='{{c_exc}} shortkey_x button_toggle_exc'>{{nums.n_excluded}} excluded</span>
             {% if arcs %}
-                <span class='{{c_par}} shortkey_p' onclick='coverage.toggle_lines(this, "par")'>{{n_par}} partial</span>
+                <span class='{{c_par}} shortkey_p button_toggle_par'>{{nums.n_partial_branches}} partial</span>
             {% endif %}
         </h2>
     </div>

python/helpers/coverage/htmlfiles/style.css

@@ -24,8 +24,8 @@ html>body {
 
 /* Set base font size to 12/16 */
 p {
-    font-size: .75em;        /* 12/16 */
-    line-height: 1.3333em;   /* 16/12 */
+    font-size: .75em;           /* 12/16 */
+    line-height: 1.33333333em;  /* 16/12 */
     }
 
 table {
@@ -102,6 +102,31 @@ h2.stats {
     border-color: #999 #ccc #ccc #999;
 }
 
+.stats span.run {
+    background: #ddffdd;
+}
+.stats span.exc {
+    background: #eeeeee;
+}
+.stats span.mis {
+    background: #ffdddd;
+}
+.stats span.hide_run {
+    background: #eeffee;
+}
+.stats span.hide_exc {
+    background: #f5f5f5;
+}
+.stats span.hide_mis {
+    background: #ffeeee;
+}
+.stats span.par {
+    background: #ffffaa;
+}
+.stats span.hide_par {
+    background: #ffffcc;
+}
+
 /* Help panel */
 #keyboard_icon {
     float: right;

python/helpers/coverage/misc.py

@@ -1,6 +1,10 @@
 """Miscellaneous stuff for Coverage."""
 
+import errno
 import inspect
+import os
+import sys
+
 from coverage.backward import md5, sorted       # pylint: disable=W0622
 from coverage.backward import string_class, to_bytes
 
@@ -34,6 +38,8 @@ def format_lines(statements, lines):
     i = 0
     j = 0
     start = None
+    statements = sorted(statements)
+    lines = sorted(lines)
     while i < len(statements) and j < len(lines):
         if statements[i] == lines[j]:
             if start == None:
@@ -50,6 +56,12 @@ def format_lines(statements, lines):
     return ret
 
 
+def short_stack():
+    """Return a string summarizing the call stack."""
+    stack = inspect.stack()[:0:-1]
+    return "\n".join(["%30s : %s @%d" % (t[3],t[1],t[2]) for t in stack])
+
+
 def expensive(fn):
     """A decorator to cache the result of an expensive operation.
 
@@ -76,13 +88,23 @@ def bool_or_none(b):
 def join_regex(regexes):
     """Combine a list of regexes into one that matches any of them."""
     if len(regexes) > 1:
-        return "(" + ")|(".join(regexes) + ")"
+        return "|".join(["(%s)" % r for r in regexes])
     elif regexes:
         return regexes[0]
     else:
         return ""
 
 
+def file_be_gone(path):
+    """Remove a file, and don't get annoyed if it doesn't exist."""
+    try:
+        os.remove(path)
+    except OSError:
+        _, e, _ = sys.exc_info()
+        if e.errno != errno.ENOENT:
+            raise
+
+
 class Hasher(object):
     """Hashes Python data into md5."""
     def __init__(self):
@@ -93,8 +115,10 @@ class Hasher(object):
         self.md5.update(to_bytes(str(type(v))))
         if isinstance(v, string_class):
             self.md5.update(to_bytes(v))
+        elif v is None:
+            pass
         elif isinstance(v, (int, float)):
-            self.update(str(v))
+            self.md5.update(to_bytes(str(v)))
         elif isinstance(v, (tuple, list)):
             for e in v:
                 self.update(e)
@@ -126,6 +150,10 @@ class NoSource(CoverageException):
     """We couldn't find the source for a module."""
     pass
 
+class NoCode(NoSource):
+    """We couldn't find any code at all."""
+    pass
+
 class NotPython(CoverageException):
     """A source file turned out not to be parsable Python."""
     pass

python/helpers/coverage/parser.py

@@ -1,9 +1,14 @@
 """Code parsing for Coverage."""
 
-import glob, opcode, os, re, sys, token, tokenize
-
+import dis
+import re
+import sys
+import token
+import tokenize
 from coverage.backward import set, sorted, StringIO # pylint: disable=W0622
-from coverage.backward import open_source
+from coverage.backward import open_source, range    # pylint: disable=W0622
+from coverage.backward import reversed              # pylint: disable=W0622
+from coverage.backward import bytes_to_ints
 from coverage.bytecode import ByteCodes, CodeObjects
 from coverage.misc import nice_pair, expensive, join_regex
 from coverage.misc import CoverageException, NoSource, NotPython
@@ -32,9 +37,13 @@ class CodeParser(object):
             except IOError:
                 _, err, _ = sys.exc_info()
                 raise NoSource(
-                    "No source for code: %r: %s" % (self.filename, err)
+                    "No source for code: '%s': %s" % (self.filename, err)
                     )
 
+        # Scrap the BOM if it exists.
+        if self.text and ord(self.text[0]) == 0xfeff:
+            self.text = self.text[1:]
+
         self.exclude = exclude
 
         self.show_tokens = False
@@ -102,9 +111,9 @@ class CodeParser(object):
         first_line = None
         empty = True
 
-        tokgen = tokenize.generate_tokens(StringIO(self.text).readline)
+        tokgen = generate_tokens(self.text)
         for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen:
-            if self.show_tokens:                # pragma: no cover
+            if self.show_tokens:                # pragma: not covered
                 print("%10s %5s %-20r %r" % (
                     tokenize.tok_name.get(toktype, toktype),
                     nice_pair((slineno, elineno)), ttext, ltext
@@ -130,8 +139,7 @@ class CodeParser(object):
                 # (a trick from trace.py in the stdlib.) This works for
                 # 99.9999% of cases.  For the rest (!) see:
                 # http://stackoverflow.com/questions/1769332/x/1769794#1769794
-                for i in range(slineno, elineno+1):
-                    self.docstrings.add(i)
+                self.docstrings.update(range(slineno, elineno+1))
             elif toktype == token.NEWLINE:
                 if first_line is not None and elineno != first_line:
                     # We're at the end of a line, and we've ended on a
@@ -170,16 +178,18 @@ class CodeParser(object):
             first_line = line
         return first_line
 
-    def first_lines(self, lines, ignore=None):
+    def first_lines(self, lines, *ignores):
         """Map the line numbers in `lines` to the correct first line of the
         statement.
 
-        Skip any line mentioned in `ignore`.
+        Skip any line mentioned in any of the sequences in `ignores`.
 
-        Returns a sorted list of the first lines.
+        Returns a set of the first lines.
 
         """
-        ignore = ignore or []
+        ignore = set()
+        for ign in ignores:
+            ignore.update(ign)
         lset = set()
         for l in lines:
             if l in ignore:
@@ -187,23 +197,34 @@ class CodeParser(object):
             new_l = self.first_line(l)
             if new_l not in ignore:
                 lset.add(new_l)
-        return sorted(lset)
+        return lset
 
     def parse_source(self):
         """Parse source text to find executable lines, excluded lines, etc.
 
-        Return values are 1) a sorted list of executable line numbers, and
-        2) a sorted list of excluded line numbers.
+        Return values are 1) a set of executable line numbers, and 2) a set of
+        excluded line numbers.
 
         Reported line numbers are normalized to the first line of multi-line
         statements.
 
         """
-        self._raw_parse()
+        try:
+            self._raw_parse()
+        except (tokenize.TokenError, IndentationError):
+            _, tokerr, _ = sys.exc_info()
+            msg, lineno = tokerr.args
+            raise NotPython(
+                "Couldn't parse '%s' as Python source: '%s' at %s" %
+                    (self.filename, msg, lineno)
+                )
 
         excluded_lines = self.first_lines(self.excluded)
-        ignore = excluded_lines + list(self.docstrings)
-        lines = self.first_lines(self.statement_starts, ignore)
+        lines = self.first_lines(
+            self.statement_starts,
+            excluded_lines,
+            self.docstrings
+        )
 
         return lines, excluded_lines
 
@@ -258,8 +279,8 @@ class CodeParser(object):
 ## Opcodes that guide the ByteParser.
 
 def _opcode(name):
-    """Return the opcode by name from the opcode module."""
-    return opcode.opmap[name]
+    """Return the opcode by name from the dis module."""
+    return dis.opmap[name]
 
 def _opcode_set(*names):
     """Return a set of opcodes by the names in `names`."""
@@ -297,7 +318,7 @@ OPS_EXCEPT_BLOCKS = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY')
 OPS_POP_BLOCK = _opcode_set('POP_BLOCK')
 
 # Opcodes that have a jump destination, but aren't really a jump.
-OPS_NO_JUMP = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY')
+OPS_NO_JUMP = OPS_PUSH_BLOCK
 
 # Individual opcodes we need below.
 OP_BREAK_LOOP = _opcode('BREAK_LOOP')
@@ -314,6 +335,7 @@ class ByteParser(object):
     def __init__(self, code=None, text=None, filename=None):
         if code:
             self.code = code
+            self.text = text
         else:
             if not text:
                 assert filename, "If no code or text, need a filename"
@@ -322,6 +344,7 @@ class ByteParser(object):
                     text = sourcef.read()
                 finally:
                     sourcef.close()
+            self.text = text
 
             try:
                 # Python 2.3 and 2.4 don't like partial last lines, so be sure
@@ -350,69 +373,54 @@ class ByteParser(object):
         The iteration includes `self` as its first value.
 
         """
-        return map(lambda c: ByteParser(code=c), CodeObjects(self.code))
-
-    # Getting numbers from the lnotab value changed in Py3.0.
-    if sys.version_info >= (3, 0):
-        def _lnotab_increments(self, lnotab):
-            """Return a list of ints from the lnotab bytes in 3.x"""
-            return list(lnotab)
-    else:
-        def _lnotab_increments(self, lnotab):
-            """Return a list of ints from the lnotab string in 2.x"""
-            return [ord(c) for c in lnotab]
+        children = CodeObjects(self.code)
+        return [ByteParser(code=c, text=self.text) for c in children]
 
     def _bytes_lines(self):
         """Map byte offsets to line numbers in `code`.
 
         Uses co_lnotab described in Python/compile.c to map byte offsets to
-        line numbers.  Returns a list: [(b0, l0), (b1, l1), ...]
+        line numbers.  Produces a sequence: (b0, l0), (b1, l1), ...
+
+        Only byte offsets that correspond to line numbers are included in the
+        results.
 
         """
         # Adapted from dis.py in the standard library.
-        byte_increments = self._lnotab_increments(self.code.co_lnotab[0::2])
-        line_increments = self._lnotab_increments(self.code.co_lnotab[1::2])
+        byte_increments = bytes_to_ints(self.code.co_lnotab[0::2])
+        line_increments = bytes_to_ints(self.code.co_lnotab[1::2])
 
-        bytes_lines = []
         last_line_num = None
         line_num = self.code.co_firstlineno
         byte_num = 0
         for byte_incr, line_incr in zip(byte_increments, line_increments):
             if byte_incr:
                 if line_num != last_line_num:
-                    bytes_lines.append((byte_num, line_num))
+                    yield (byte_num, line_num)
                     last_line_num = line_num
                 byte_num += byte_incr
             line_num += line_incr
         if line_num != last_line_num:
-            bytes_lines.append((byte_num, line_num))
-        return bytes_lines
+            yield (byte_num, line_num)
 
     def _find_statements(self):
         """Find the statements in `self.code`.
 
-        Return a set of line numbers that start statements.  Recurses into all
-        code objects reachable from `self.code`.
+        Produce a sequence of line numbers that start statements.  Recurses
+        into all code objects reachable from `self.code`.
 
         """
-        stmts = set()
         for bp in self.child_parsers():
             # Get all of the lineno information from this code.
             for _, l in bp._bytes_lines():
-                stmts.add(l)
-        return stmts
+                yield l
 
-    def _disassemble(self):     # pragma: no cover
-        """Disassemble code, for ad-hoc experimenting."""
-
-        import dis
-
-        for bp in self.child_parsers():
-            print("\n%s: " % bp.code)
-            dis.dis(bp.code)
-            print("Bytes lines: %r" % bp._bytes_lines())
-
-        print("")
+    def _block_stack_repr(self, block_stack):
+        """Get a string version of `block_stack`, for debugging."""
+        blocks = ", ".join(
+            ["(%s, %r)" % (dis.opname[b[0]], b[1]) for b in block_stack]
+        )
+        return "[" + blocks + "]"
 
     def _split_into_chunks(self):
         """Split the code object into a list of `Chunk` objects.
@@ -423,10 +431,11 @@ class ByteParser(object):
         Returns a list of `Chunk` objects.
 
         """
-
         # The list of chunks so far, and the one we're working on.
         chunks = []
         chunk = None
+
+        # A dict mapping byte offsets of line starts to the line numbers.
         bytes_lines_map = dict(self._bytes_lines())
 
         # The block stack: loops and try blocks get pushed here for the
@@ -441,24 +450,38 @@ class ByteParser(object):
         # We have to handle the last two bytecodes specially.
         ult = penult = None
 
-        for bc in ByteCodes(self.code.co_code):
+        # Get a set of all of the jump-to points.
+        jump_to = set()
+        bytecodes = list(ByteCodes(self.code.co_code))
+        for bc in bytecodes:
+            if bc.jump_to >= 0:
+                jump_to.add(bc.jump_to)
+
+        chunk_lineno = 0
+
+        # Walk the byte codes building chunks.
+        for bc in bytecodes:
             # Maybe have to start a new chunk
+            start_new_chunk = False
+            first_chunk = False
             if bc.offset in bytes_lines_map:
                 # Start a new chunk for each source line number.
-                if chunk:
-                    chunk.exits.add(bc.offset)
-                chunk = Chunk(bc.offset, bytes_lines_map[bc.offset])
-                chunks.append(chunk)
+                start_new_chunk = True
+                chunk_lineno = bytes_lines_map[bc.offset]
+                first_chunk = True
+            elif bc.offset in jump_to:
+                # To make chunks have a single entrance, we have to make a new
+                # chunk when we get to a place some bytecode jumps to.
+                start_new_chunk = True
             elif bc.op in OPS_CHUNK_BEGIN:
                 # Jumps deserve their own unnumbered chunk.  This fixes
                 # problems with jumps to jumps getting confused.
+                start_new_chunk = True
+
+            if not chunk or start_new_chunk:
                 if chunk:
                     chunk.exits.add(bc.offset)
-                chunk = Chunk(bc.offset)
-                chunks.append(chunk)
-
-            if not chunk:
-                chunk = Chunk(bc.offset)
+                chunk = Chunk(bc.offset, chunk_lineno, first_chunk)
                 chunks.append(chunk)
 
             # Look at the opcode
@@ -487,15 +510,11 @@ class ByteParser(object):
                     chunk.exits.add(block_stack[-1][1])
                 chunk = None
             if bc.op == OP_END_FINALLY:
-                if block_stack:
-                    # A break that goes through a finally will jump to whatever
-                    # block is on top of the stack.
-                    chunk.exits.add(block_stack[-1][1])
                 # For the finally clause we need to find the closest exception
                 # block, and use its jump target as an exit.
-                for iblock in range(len(block_stack)-1, -1, -1):
-                    if block_stack[iblock][0] in OPS_EXCEPT_BLOCKS:
-                        chunk.exits.add(block_stack[iblock][1])
+                for block in reversed(block_stack):
+                    if block[0] in OPS_EXCEPT_BLOCKS:
+                        chunk.exits.add(block[1])
                         break
             if bc.op == OP_COMPARE_OP and bc.arg == COMPARE_EXCEPTION:
                 # This is an except clause.  We want to overlook the next
@@ -521,23 +540,33 @@ class ByteParser(object):
                             last_chunk = chunks[-1]
                             last_chunk.exits.remove(ex)
                             last_chunk.exits.add(penult.offset)
-                            chunk = Chunk(penult.offset)
+                            chunk = Chunk(
+                                penult.offset, last_chunk.line, False
+                            )
                             chunk.exits.add(ex)
                             chunks.append(chunk)
 
             # Give all the chunks a length.
-            chunks[-1].length = bc.next_offset - chunks[-1].byte
+            chunks[-1].length = bc.next_offset - chunks[-1].byte # pylint: disable=W0631,C0301
             for i in range(len(chunks)-1):
                 chunks[i].length = chunks[i+1].byte - chunks[i].byte
 
+        #self.validate_chunks(chunks)
         return chunks
 
+    def validate_chunks(self, chunks):
+        """Validate the rule that chunks have a single entrance."""
+        # starts is the entrances to the chunks
+        starts = set([ch.byte for ch in chunks])
+        for ch in chunks:
+            assert all([(ex in starts or ex < 0) for ex in ch.exits])
+
     def _arcs(self):
         """Find the executable arcs in the code.
 
-        Returns a set of pairs, (from,to).  From and to are integer line
-        numbers.  If from is < 0, then the arc is an entrance into the code
-        object.  If to is < 0, the arc is an exit from the code object.
+        Yields pairs: (from,to).  From and to are integer line numbers.  If
+        from is < 0, then the arc is an entrance into the code object.  If to
+        is < 0, the arc is an exit from the code object.
 
         """
         chunks = self._split_into_chunks()
@@ -545,65 +574,43 @@ class ByteParser(object):
         # A map from byte offsets to chunks jumped into.
         byte_chunks = dict([(c.byte, c) for c in chunks])
 
-        # Build a map from byte offsets to actual lines reached.
-        byte_lines = {}
-        bytes_to_add = set([c.byte for c in chunks])
+        # There's always an entrance at the first chunk.
+        yield (-1, byte_chunks[0].line)
 
-        while bytes_to_add:
-            byte_to_add = bytes_to_add.pop()
-            if byte_to_add in byte_lines or byte_to_add < 0:
+        # Traverse from the first chunk in each line, and yield arcs where
+        # the trace function will be invoked.
+        for chunk in chunks:
+            if not chunk.first:
                 continue
 
-            # Which lines does this chunk lead to?
-            bytes_considered = set()
-            bytes_to_consider = [byte_to_add]
-            lines = set()
+            chunks_considered = set()
+            chunks_to_consider = [chunk]
+            while chunks_to_consider:
+                # Get the chunk we're considering, and make sure we don't
+                # consider it again
+                this_chunk = chunks_to_consider.pop()
+                chunks_considered.add(this_chunk)
 
-            while bytes_to_consider:
-                byte = bytes_to_consider.pop()
-                bytes_considered.add(byte)
-
-                # Find chunk for byte
-                try:
-                    ch = byte_chunks[byte]
-                except KeyError:
-                    for ch in chunks:
-                        if ch.byte <= byte < ch.byte+ch.length:
-                            break
-                    else:
-                        # No chunk for this byte!
-                        raise Exception("Couldn't find chunk @ %d" % byte)
-                    byte_chunks[byte] = ch
-
-                if ch.line:
-                    lines.add(ch.line)
-                else:
-                    for ex in ch.exits:
-                        if ex < 0:
-                            lines.add(ex)
-                        elif ex not in bytes_considered:
-                            bytes_to_consider.append(ex)
-
-                bytes_to_add.update(ch.exits)
-
-            byte_lines[byte_to_add] = lines
-
-        # Figure out for each chunk where the exits go.
-        arcs = set()
-        for chunk in chunks:
-            if chunk.line:
-                for ex in chunk.exits:
+                # For each exit, add the line number if the trace function
+                # would be triggered, or add the chunk to those being
+                # considered if not.
+                for ex in this_chunk.exits:
                     if ex < 0:
-                        exit_lines = [ex]
+                        yield (chunk.line, ex)
                     else:
-                        exit_lines = byte_lines[ex]
-                    for exit_line in exit_lines:
-                        if chunk.line != exit_line:
-                            arcs.add((chunk.line, exit_line))
-        for line in byte_lines[0]:
-            arcs.add((-1, line))
+                        next_chunk = byte_chunks[ex]
+                        if next_chunk in chunks_considered:
+                            continue
 
-        return arcs
+                        # The trace function is invoked if visiting the first
+                        # bytecode in a line, or if the transition is a
+                        # backward jump.
+                        backward_jump = next_chunk.byte < this_chunk.byte
+                        if next_chunk.first or backward_jump:
+                            if next_chunk.line != chunk.line:
+                                yield (chunk.line, next_chunk.line)
+                        else:
+                            chunks_to_consider.append(next_chunk)
 
     def _all_chunks(self):
         """Returns a list of `Chunk` objects for this code and its children.
@@ -631,11 +638,11 @@ class ByteParser(object):
 
 
 class Chunk(object):
-    """A sequence of bytecodes with a single entrance.
+    """A sequence of byte codes with a single entrance.
 
     To analyze byte code, we have to divide it into chunks, sequences of byte
-    codes such that each basic block has only one entrance, the first
-    instruction in the block.
+    codes such that each chunk has only one entrance, the first instruction in
+    the block.
 
     This is almost the CS concept of `basic block`_, except that we're willing
     to have many exits from a chunk, and "basic block" is a more cumbersome
@@ -643,158 +650,54 @@ class Chunk(object):
 
     .. _basic block: http://en.wikipedia.org/wiki/Basic_block
 
+    `line` is the source line number containing this chunk.
+
+    `first` is true if this is the first chunk in the source line.
+
     An exit < 0 means the chunk can leave the code (return).  The exit is
     the negative of the starting line number of the code block.
 
     """
-    def __init__(self, byte, line=0):
+    def __init__(self, byte, line, first):
         self.byte = byte
         self.line = line
+        self.first = first
         self.length = 0
         self.exits = set()
 
     def __repr__(self):
-        return "<%d+%d @%d %r>" % (
-            self.byte, self.length, self.line, list(self.exits)
-            )
-
-
-class AdHocMain(object):        # pragma: no cover
-    """An ad-hoc main for code parsing experiments."""
-
-    def main(self, args):
-        """A main function for trying the code from the command line."""
-
-        from optparse import OptionParser
-
-        parser = OptionParser()
-        parser.add_option(
-            "-c", action="store_true", dest="chunks",
-            help="Show basic block chunks"
-            )
-        parser.add_option(
-            "-d", action="store_true", dest="dis",
-            help="Disassemble"
-            )
-        parser.add_option(
-            "-R", action="store_true", dest="recursive",
-            help="Recurse to find source files"
-            )
-        parser.add_option(
-            "-s", action="store_true", dest="source",
-            help="Show analyzed source"
-            )
-        parser.add_option(
-            "-t", action="store_true", dest="tokens",
-            help="Show tokens"
-            )
-
-        options, args = parser.parse_args()
-        if options.recursive:
-            if args:
-                root = args[0]
-            else:
-                root = "."
-            for root, _, _ in os.walk(root):
-                for f in glob.glob(root + "/*.py"):
-                    self.adhoc_one_file(options, f)
+        if self.first:
+            bang = "!"
         else:
-            self.adhoc_one_file(options, args[0])
+            bang = ""
+        return "<%d+%d @%d%s %r>" % (
+            self.byte, self.length, self.line, bang, list(self.exits)
+            )
 
-    def adhoc_one_file(self, options, filename):
-        """Process just one file."""
 
-        if options.dis or options.chunks:
-            try:
-                bp = ByteParser(filename=filename)
-            except CoverageException:
-                _, err, _ = sys.exc_info()
-                print("%s" % (err,))
-                return
+class CachedTokenizer(object):
+    """A one-element cache around tokenize.generate_tokens.
 
-        if options.dis:
-            print("Main code:")
-            bp._disassemble()
+    When reporting, coverage.py tokenizes files twice, once to find the
+    structure of the file, and once to syntax-color it.  Tokenizing is
+    expensive, and easily cached.
 
-        if options.chunks:
-            chunks = bp._all_chunks()
-            if options.recursive:
-                print("%6d: %s" % (len(chunks), filename))
-            else:
-                print("Chunks: %r" % chunks)
-                arcs = bp._all_arcs()
-                print("Arcs: %r" % sorted(arcs))
+    This is a one-element cache so that our twice-in-a-row tokenizing doesn't
+    actually tokenize twice.
 
-        if options.source or options.tokens:
-            cp = CodeParser(filename=filename, exclude=r"no\s*cover")
-            cp.show_tokens = options.tokens
-            cp._raw_parse()
+    """
+    def __init__(self):
+        self.last_text = None
+        self.last_tokens = None
 
-            if options.source:
-                if options.chunks:
-                    arc_width, arc_chars = self.arc_ascii_art(arcs)
-                else:
-                    arc_width, arc_chars = 0, {}
+    def generate_tokens(self, text):
+        """A stand-in for `tokenize.generate_tokens`."""
+        if text != self.last_text:
+            self.last_text = text
+            self.last_tokens = list(
+                tokenize.generate_tokens(StringIO(text).readline)
+            )
+        return self.last_tokens
 
-                exit_counts = cp.exit_counts()
-
-                for i, ltext in enumerate(cp.lines):
-                    lineno = i+1
-                    m0 = m1 = m2 = m3 = a = ' '
-                    if lineno in cp.statement_starts:
-                        m0 = '-'
-                    exits = exit_counts.get(lineno, 0)
-                    if exits > 1:
-                        m1 = str(exits)
-                    if lineno in cp.docstrings:
-                        m2 = '"'
-                    if lineno in cp.classdefs:
-                        m2 = 'C'
-                    if lineno in cp.excluded:
-                        m3 = 'x'
-                    a = arc_chars.get(lineno, '').ljust(arc_width)
-                    print("%4d %s%s%s%s%s %s" %
-                                (lineno, m0, m1, m2, m3, a, ltext)
-                        )
-
-    def arc_ascii_art(self, arcs):
-        """Draw arcs as ascii art.
-
-        Returns a width of characters needed to draw all the arcs, and a
-        dictionary mapping line numbers to ascii strings to draw for that line.
-
-        """
-        arc_chars = {}
-        for lfrom, lto in sorted(arcs):
-            if lfrom < 0:
-                arc_chars[lto] = arc_chars.get(lto, '') + 'v'
-            elif lto < 0:
-                arc_chars[lfrom] = arc_chars.get(lfrom, '') + '^'
-            else:
-                if lfrom == lto - 1:
-                    # Don't show obvious arcs.
-                    continue
-                if lfrom < lto:
-                    l1, l2 = lfrom, lto
-                else:
-                    l1, l2 = lto, lfrom
-                w = max([len(arc_chars.get(l, '')) for l in range(l1, l2+1)])
-                for l in range(l1, l2+1):
-                    if l == lfrom:
-                        ch = '<'
-                    elif l == lto:
-                        ch = '>'
-                    else:
-                        ch = '|'
-                    arc_chars[l] = arc_chars.get(l, '').ljust(w) + ch
-                arc_width = 0
-
-        if arc_chars:
-            arc_width = max([len(a) for a in arc_chars.values()])
-        else:
-            arc_width = 0
-
-        return arc_width, arc_chars
-
-if __name__ == '__main__':
-    AdHocMain().main(sys.argv[1:])
+# Create our generate_tokens cache as a callable replacement function.
+generate_tokens = CachedTokenizer().generate_tokens

python/helpers/coverage/phystokens.py

@@ -1,7 +1,14 @@
 """Better tokenizing for coverage.py."""
 
-import keyword, re, token, tokenize
-from coverage.backward import StringIO              # pylint: disable=W0622
+import codecs
+import keyword
+import re
+import sys
+import token
+import tokenize
+from coverage.backward import set                       # pylint: disable=W0622
+from coverage.parser import generate_tokens
+
 
 def phys_tokens(toks):
     """Return all physical tokens, even line continuations.
@@ -18,7 +25,7 @@ def phys_tokens(toks):
     last_ttype = None
     for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks:
         if last_lineno != elineno:
-            if last_line and last_line[-2:] == "\\\n":
+            if last_line and last_line.endswith("\\\n"):
                 # We are at the beginning of a new line, and the last line
                 # ended with a backslash.  We probably have to inject a
                 # backslash token into the stream. Unfortunately, there's more
@@ -74,11 +81,11 @@ def source_token_lines(source):
     is indistinguishable from a final line with a newline.
 
     """
-    ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL]
+    ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
     line = []
     col = 0
     source = source.expandtabs(8).replace('\r\n', '\n')
-    tokgen = tokenize.generate_tokens(StringIO(source).readline)
+    tokgen = generate_tokens(source)
     for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
         mark_start = True
         for part in re.split('(\n)', ttext):
@@ -106,3 +113,103 @@ def source_token_lines(source):
 
     if line:
         yield line
+
+def source_encoding(source):
+    """Determine the encoding for `source` (a string), according to PEP 263.
+
+    Returns a string, the name of the encoding.
+
+    """
+    # Note: this function should never be called on Python 3, since py3 has
+    # built-in tools to do this.
+    assert sys.version_info < (3, 0)
+
+    # This is mostly code adapted from Py3.2's tokenize module.
+
+    cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)")
+
+    # Do this so the detect_encode code we copied will work.
+    readline = iter(source.splitlines(True)).next
+
+    def _get_normal_name(orig_enc):
+        """Imitates get_normal_name in tokenizer.c."""
+        # Only care about the first 12 characters.
+        enc = orig_enc[:12].lower().replace("_", "-")
+        if re.match(r"^utf-8($|-)", enc):
+            return "utf-8"
+        if re.match(r"^(latin-1|iso-8859-1|iso-latin-1)($|-)", enc):
+            return "iso-8859-1"
+        return orig_enc
+
+    # From detect_encode():
+    # It detects the encoding from the presence of a utf-8 bom or an encoding
+    # cookie as specified in pep-0263.  If both a bom and a cookie are present,
+    # but disagree, a SyntaxError will be raised.  If the encoding cookie is an
+    # invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
+    # 'utf-8-sig' is returned.
+
+    # If no encoding is specified, then the default will be returned.  The
+    # default varied with version.
+
+    if sys.version_info <= (2, 4):
+        default = 'iso-8859-1'
+    else:
+        default = 'ascii'
+
+    bom_found = False
+    encoding = None
+
+    def read_or_stop():
+        """Get the next source line, or ''."""
+        try:
+            return readline()
+        except StopIteration:
+            return ''
+
+    def find_cookie(line):
+        """Find an encoding cookie in `line`."""
+        try:
+            line_string = line.decode('ascii')
+        except UnicodeDecodeError:
+            return None
+
+        matches = cookie_re.findall(line_string)
+        if not matches:
+            return None
+        encoding = _get_normal_name(matches[0])
+        try:
+            codec = codecs.lookup(encoding)
+        except LookupError:
+            # This behaviour mimics the Python interpreter
+            raise SyntaxError("unknown encoding: " + encoding)
+
+        if bom_found:
+            # codecs in 2.3 were raw tuples of functions, assume the best.
+            codec_name = getattr(codec, 'name', encoding)
+            if codec_name != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            encoding += '-sig'
+        return encoding
+
+    first = read_or_stop()
+    if first.startswith(codecs.BOM_UTF8):
+        bom_found = True
+        first = first[3:]
+        default = 'utf-8-sig'
+    if not first:
+        return default
+
+    encoding = find_cookie(first)
+    if encoding:
+        return encoding
+
+    second = read_or_stop()
+    if not second:
+        return default
+
+    encoding = find_cookie(second)
+    if encoding:
+        return encoding
+
+    return default

python/helpers/coverage/report.py

@@ -1,21 +1,24 @@
 """Reporter foundation for Coverage."""
 
-import fnmatch, os
+import fnmatch
+import os
 from coverage.codeunit import code_unit_factory
+from coverage.files import prep_patterns
 from coverage.misc import CoverageException, NoSource, NotPython
 
+
 class Reporter(object):
     """A base class for all reporters."""
 
-    def __init__(self, coverage, ignore_errors=False):
+    def __init__(self, coverage, config):
         """Create a reporter.
 
-        `coverage` is the coverage instance. `ignore_errors` controls how
-        skittish the reporter will be during file processing.
+        `coverage` is the coverage instance. `config` is an instance  of
+        CoverageConfig, for controlling all sorts of behavior.
 
         """
         self.coverage = coverage
-        self.ignore_errors = ignore_errors
+        self.config = config
 
         # The code units to report on.  Set by find_code_units.
         self.code_units = []
@@ -24,19 +27,18 @@ class Reporter(object):
         # classes.
         self.directory = None
 
-    def find_code_units(self, morfs, config):
+    def find_code_units(self, morfs):
         """Find the code units we'll report on.
 
-        `morfs` is a list of modules or filenames. `config` is a
-        CoverageConfig instance.
+        `morfs` is a list of modules or filenames.
 
         """
         morfs = morfs or self.coverage.data.measured_files()
         file_locator = self.coverage.file_locator
         self.code_units = code_unit_factory(morfs, file_locator)
 
-        if config.include:
-            patterns = [file_locator.abs_file(p) for p in config.include]
+        if self.config.include:
+            patterns = prep_patterns(self.config.include)
             filtered = []
             for cu in self.code_units:
                 for pattern in patterns:
@@ -45,8 +47,8 @@ class Reporter(object):
                         break
             self.code_units = filtered
 
-        if config.omit:
-            patterns = [file_locator.abs_file(p) for p in config.omit]
+        if self.config.omit:
+            patterns = prep_patterns(self.config.omit)
             filtered = []
             for cu in self.code_units:
                 for pattern in patterns:
@@ -58,7 +60,7 @@ class Reporter(object):
 
         self.code_units.sort()
 
-    def report_files(self, report_fn, morfs, config, directory=None):
+    def report_files(self, report_fn, morfs, directory=None):
         """Run a reporting function on a number of morfs.
 
         `report_fn` is called for each relative morf in `morfs`.  It is called
@@ -69,10 +71,8 @@ class Reporter(object):
         where `code_unit` is the `CodeUnit` for the morf, and `analysis` is
         the `Analysis` for the morf.
 
-        `config` is a CoverageConfig instance.
-
         """
-        self.find_code_units(morfs, config)
+        self.find_code_units(morfs)
 
         if not self.code_units:
             raise CoverageException("No data to report.")
@@ -84,6 +84,11 @@ class Reporter(object):
         for cu in self.code_units:
             try:
                 report_fn(cu, self.coverage._analyze(cu))
-            except (NoSource, NotPython):
-                if not self.ignore_errors:
+            except NoSource:
+                if not self.config.ignore_errors:
+                    raise
+            except NotPython:
+                # Only report errors for .py files, and only if we didn't
+                # explicitly suppress those errors.
+                if cu.should_be_python() and not self.config.ignore_errors:
                     raise

python/helpers/coverage/results.py

@@ -2,7 +2,7 @@
 
 import os
 
-from coverage.backward import set, sorted           # pylint: disable=W0622
+from coverage.backward import iitems, set, sorted       # pylint: disable=W0622
 from coverage.misc import format_lines, join_regex, NoSource
 from coverage.parser import CodeParser
 
@@ -15,16 +15,10 @@ class Analysis(object):
         self.code_unit = code_unit
 
         self.filename = self.code_unit.filename
-        ext = os.path.splitext(self.filename)[1]
-        source = None
-        if ext == '.py':
-            if not os.path.exists(self.filename):
-                source = self.coverage.file_locator.get_zip_data(self.filename)
-                if not source:
-                    raise NoSource("No source for code: %r" % self.filename)
+        actual_filename, source = self.find_source(self.filename)
 
         self.parser = CodeParser(
-            text=source, filename=self.filename,
+            text=source, filename=actual_filename,
             exclude=self.coverage._exclude_regex('exclude')
             )
         self.statements, self.excluded = self.parser.parse_source()
@@ -32,7 +26,7 @@ class Analysis(object):
         # Identify missing statements.
         executed = self.coverage.data.executed_lines(self.filename)
         exec1 = self.parser.first_lines(executed)
-        self.missing = sorted(set(self.statements) - set(exec1))
+        self.missing = self.statements - exec1
 
         if self.coverage.data.has_arcs():
             self.no_branch = self.parser.lines_matching(
@@ -41,9 +35,12 @@ class Analysis(object):
                 )
             n_branches = self.total_branches()
             mba = self.missing_branch_arcs()
-            n_missing_branches = sum([len(v) for v in mba.values()])
+            n_partial_branches = sum(
+                [len(v) for k,v in iitems(mba) if k not in self.missing]
+                )
+            n_missing_branches = sum([len(v) for k,v in iitems(mba)])
         else:
-            n_branches = n_missing_branches = 0
+            n_branches = n_partial_branches = n_missing_branches = 0
             self.no_branch = set()
 
         self.numbers = Numbers(
@@ -52,9 +49,48 @@ class Analysis(object):
             n_excluded=len(self.excluded),
             n_missing=len(self.missing),
             n_branches=n_branches,
+            n_partial_branches=n_partial_branches,
             n_missing_branches=n_missing_branches,
             )
 
+    def find_source(self, filename):
+        """Find the source for `filename`.
+
+        Returns two values: the actual filename, and the source.
+
+        The source returned depends on which of these cases holds:
+
+            * The filename seems to be a non-source file: returns None
+
+            * The filename is a source file, and actually exists: returns None.
+
+            * The filename is a source file, and is in a zip file or egg:
+              returns the source.
+
+            * The filename is a source file, but couldn't be found: raises
+              `NoSource`.
+
+        """
+        source = None
+
+        base, ext = os.path.splitext(filename)
+        TRY_EXTS = {
+            '.py':  ['.py', '.pyw'],
+            '.pyw': ['.pyw'],
+        }
+        try_exts = TRY_EXTS.get(ext)
+        if not try_exts:
+            return filename, None
+
+        for try_ext in try_exts:
+            try_filename = base + try_ext
+            if os.path.exists(try_filename):
+                return try_filename, None
+            source = self.coverage.file_locator.get_zip_data(try_filename)
+            if source:
+                return try_filename, source
+        raise NoSource("No source for code: '%s'" % filename)
+
     def missing_formatted(self):
         """The missing line numbers, formatted nicely.
 
@@ -107,7 +143,7 @@ class Analysis(object):
     def branch_lines(self):
         """Returns a list of line numbers that have more than one exit."""
         exit_counts = self.parser.exit_counts()
-        return [l1 for l1,count in exit_counts.items() if count > 1]
+        return [l1 for l1,count in iitems(exit_counts) if count > 1]
 
     def total_branches(self):
         """How many total branches are there?"""
@@ -164,13 +200,14 @@ class Numbers(object):
     _near100 = 99.0
 
     def __init__(self, n_files=0, n_statements=0, n_excluded=0, n_missing=0,
-                    n_branches=0, n_missing_branches=0
+                    n_branches=0, n_partial_branches=0, n_missing_branches=0
                     ):
         self.n_files = n_files
         self.n_statements = n_statements
         self.n_excluded = n_excluded
         self.n_missing = n_missing
         self.n_branches = n_branches
+        self.n_partial_branches = n_partial_branches
         self.n_missing_branches = n_missing_branches
 
     def set_precision(cls, precision):
@@ -234,8 +271,12 @@ class Numbers(object):
         nums.n_excluded = self.n_excluded + other.n_excluded
         nums.n_missing = self.n_missing + other.n_missing
         nums.n_branches = self.n_branches + other.n_branches
-        nums.n_missing_branches = (self.n_missing_branches +
-                                                    other.n_missing_branches)
+        nums.n_partial_branches = (
+            self.n_partial_branches + other.n_partial_branches
+            )
+        nums.n_missing_branches = (
+            self.n_missing_branches + other.n_missing_branches
+            )
         return nums
 
     def __radd__(self, other):

python/helpers/coverage/summary.py

@@ -4,24 +4,23 @@ import sys
 
 from coverage.report import Reporter
 from coverage.results import Numbers
+from coverage.misc import NotPython
 
 
 class SummaryReporter(Reporter):
     """A reporter for writing the summary report."""
 
-    def __init__(self, coverage, show_missing=True, ignore_errors=False):
-        super(SummaryReporter, self).__init__(coverage, ignore_errors)
-        self.show_missing = show_missing
+    def __init__(self, coverage, config):
+        super(SummaryReporter, self).__init__(coverage, config)
         self.branches = coverage.data.has_arcs()
 
-    def report(self, morfs, outfile=None, config=None):
+    def report(self, morfs, outfile=None):
         """Writes a report summarizing coverage statistics per module.
 
-        `outfile` is a file object to write the summary to.  `config` is a
-        CoverageConfig instance.
+        `outfile` is a file object to write the summary to.
 
         """
-        self.find_code_units(morfs, config)
+        self.find_code_units(morfs)
 
         # Prepare the formatting strings
         max_name = max([len(cu.name) for cu in self.code_units] + [5])
@@ -30,12 +29,12 @@ class SummaryReporter(Reporter):
         header = (fmt_name % "Name") + " Stmts   Miss"
         fmt_coverage = fmt_name + "%6d %6d"
         if self.branches:
-            header += " Branch BrPart"
+            header += " Branch BrMiss"
             fmt_coverage += " %6d %6d"
         width100 = Numbers.pc_str_width()
         header += "%*s" % (width100+4, "Cover")
         fmt_coverage += "%%%ds%%%%" % (width100+3,)
-        if self.show_missing:
+        if self.config.show_missing:
             header += "   Missing"
             fmt_coverage += "   %s"
         rule = "-" * len(header) + "\n"
@@ -59,15 +58,19 @@ class SummaryReporter(Reporter):
                 if self.branches:
                     args += (nums.n_branches, nums.n_missing_branches)
                 args += (nums.pc_covered_str,)
-                if self.show_missing:
+                if self.config.show_missing:
                     args += (analysis.missing_formatted(),)
                 outfile.write(fmt_coverage % args)
                 total += nums
-            except KeyboardInterrupt:                       # pragma: no cover
+            except KeyboardInterrupt:                   # pragma: not covered
                 raise
             except:
-                if not self.ignore_errors:
+                report_it = not self.config.ignore_errors
+                if report_it:
                     typ, msg = sys.exc_info()[:2]
+                    if typ is NotPython and not cu.should_be_python():
+                        report_it = False
+                if report_it:
                     outfile.write(fmt_err % (cu.name, typ.__name__, msg))
 
         if total.n_files > 1:
@@ -76,6 +79,8 @@ class SummaryReporter(Reporter):
             if self.branches:
                 args += (total.n_branches, total.n_missing_branches)
             args += (total.pc_covered_str,)
-            if self.show_missing:
+            if self.config.show_missing:
                 args += ("",)
             outfile.write(fmt_coverage % args)
+
+        return total.pc_covered

python/helpers/coverage/templite.py

@@ -2,7 +2,53 @@
 
 # Coincidentally named the same as http://code.activestate.com/recipes/496702/
 
-import re, sys
+import re
+
+from coverage.backward import set                       # pylint: disable=W0622
+
+
+class CodeBuilder(object):
+    """Build source code conveniently."""
+
+    def __init__(self, indent=0):
+        self.code = []
+        self.indent_amount = indent
+
+    def add_line(self, line):
+        """Add a line of source to the code.
+
+        Don't include indentations or newlines.
+
+        """
+        self.code.append(" " * self.indent_amount)
+        self.code.append(line)
+        self.code.append("\n")
+
+    def add_section(self):
+        """Add a section, a sub-CodeBuilder."""
+        sect = CodeBuilder(self.indent_amount)
+        self.code.append(sect)
+        return sect
+
+    def indent(self):
+        """Increase the current indent for following lines."""
+        self.indent_amount += 4
+
+    def dedent(self):
+        """Decrease the current indent for following lines."""
+        self.indent_amount -= 4
+
+    def __str__(self):
+        return "".join([str(c) for c in self.code])
+
+    def get_function(self, fn_name):
+        """Compile the code, and return the function `fn_name`."""
+        assert self.indent_amount == 0
+        g = {}
+        code_text = str(self)
+        exec(code_text, g)
+        return g[fn_name]
+
 
 class Templite(object):
     """A simple template renderer, for a nano-subset of Django syntax.
@@ -39,53 +85,104 @@ class Templite(object):
         for context in contexts:
             self.context.update(context)
 
+        # We construct a function in source form, then compile it and hold onto
+        # it, and execute it to render the template.
+        code = CodeBuilder()
+
+        code.add_line("def render(ctx, dot):")
+        code.indent()
+        vars_code = code.add_section()
+        self.all_vars = set()
+        self.loop_vars = set()
+        code.add_line("result = []")
+        code.add_line("a = result.append")
+        code.add_line("e = result.extend")
+        code.add_line("s = str")
+
+        buffered = []
+        def flush_output():
+            """Force `buffered` to the code builder."""
+            if len(buffered) == 1:
+                code.add_line("a(%s)" % buffered[0])
+            elif len(buffered) > 1:
+                code.add_line("e([%s])" % ",".join(buffered))
+            del buffered[:]
+
         # Split the text to form a list of tokens.
         toks = re.split(r"(?s)({{.*?}}|{%.*?%}|{#.*?#})", text)
 
-        # Parse the tokens into a nested list of operations.  Each item in the
-        # list is a tuple with an opcode, and arguments.  They'll be
-        # interpreted by TempliteEngine.
-        #
-        # When parsing an action tag with nested content (if, for), the current
-        # ops list is pushed onto ops_stack, and the parsing continues in a new
-        # ops list that is part of the arguments to the if or for op.
-        ops = []
         ops_stack = []
         for tok in toks:
             if tok.startswith('{{'):
-                # Expression: ('exp', expr)
-                ops.append(('exp', tok[2:-2].strip()))
+                # An expression to evaluate.
+                buffered.append("s(%s)" % self.expr_code(tok[2:-2].strip()))
             elif tok.startswith('{#'):
                 # Comment: ignore it and move on.
                 continue
             elif tok.startswith('{%'):
                 # Action tag: split into words and parse further.
+                flush_output()
                 words = tok[2:-2].strip().split()
                 if words[0] == 'if':
-                    # If: ('if', (expr, body_ops))
-                    if_ops = []
+                    # An if statement: evaluate the expression to determine if.
                     assert len(words) == 2
-                    ops.append(('if', (words[1], if_ops)))
-                    ops_stack.append(ops)
-                    ops = if_ops
+                    ops_stack.append('if')
+                    code.add_line("if %s:" % self.expr_code(words[1]))
+                    code.indent()
                 elif words[0] == 'for':
-                    # For: ('for', (varname, listexpr, body_ops))
+                    # A loop: iterate over expression result.
                     assert len(words) == 4 and words[2] == 'in'
-                    for_ops = []
-                    ops.append(('for', (words[1], words[3], for_ops)))
-                    ops_stack.append(ops)
-                    ops = for_ops
+                    ops_stack.append('for')
+                    self.loop_vars.add(words[1])
+                    code.add_line(
+                        "for c_%s in %s:" % (
+                            words[1],
+                            self.expr_code(words[3])
+                        )
+                    )
+                    code.indent()
                 elif words[0].startswith('end'):
                     # Endsomething.  Pop the ops stack
-                    ops = ops_stack.pop()
-                    assert ops[-1][0] == words[0][3:]
+                    end_what = words[0][3:]
+                    if ops_stack[-1] != end_what:
+                        raise SyntaxError("Mismatched end tag: %r" % end_what)
+                    ops_stack.pop()
+                    code.dedent()
                 else:
-                    raise SyntaxError("Don't understand tag %r" % words)
+                    raise SyntaxError("Don't understand tag: %r" % words[0])
             else:
-                ops.append(('lit', tok))
+                # Literal content.  If it isn't empty, output it.
+                if tok:
+                    buffered.append("%r" % tok)
+        flush_output()
 
-        assert not ops_stack, "Unmatched action tag: %r" % ops_stack[-1][0]
-        self.ops = ops
+        for var_name in self.all_vars - self.loop_vars:
+            vars_code.add_line("c_%s = ctx[%r]" % (var_name, var_name))
+
+        if ops_stack:
+            raise SyntaxError("Unmatched action tag: %r" % ops_stack[-1])
+
+        code.add_line("return ''.join(result)")
+        code.dedent()
+        self.render_function = code.get_function('render')
+
+    def expr_code(self, expr):
+        """Generate a Python expression for `expr`."""
+        if "|" in expr:
+            pipes = expr.split("|")
+            code = self.expr_code(pipes[0])
+            for func in pipes[1:]:
+                self.all_vars.add(func)
+                code = "c_%s(%s)" % (func, code)
+        elif "." in expr:
+            dots = expr.split(".")
+            code = self.expr_code(dots[0])
+            args = [repr(d) for d in dots[1:]]
+            code = "dot(%s, %s)" % (code, ", ".join(args))
+        else:
+            self.all_vars.add(expr)
+            code = "c_%s" % expr
+        return code
 
     def render(self, context=None):
         """Render this template by applying it to `context`.
@@ -97,70 +194,15 @@ class Templite(object):
         ctx = dict(self.context)
         if context:
             ctx.update(context)
+        return self.render_function(ctx, self.do_dots)
 
-        # Run it through an engine, and return the result.
-        engine = _TempliteEngine(ctx)
-        engine.execute(self.ops)
-        return "".join(engine.result)
-
-
-class _TempliteEngine(object):
-    """Executes Templite objects to produce strings."""
-    def __init__(self, context):
-        self.context = context
-        self.result = []
-
-    def execute(self, ops):
-        """Execute `ops` in the engine.
-
-        Called recursively for the bodies of if's and loops.
-
-        """
-        for op, args in ops:
-            if op == 'lit':
-                self.result.append(args)
-            elif op == 'exp':
-                try:
-                    self.result.append(str(self.evaluate(args)))
-                except:
-                    exc_class, exc, _ = sys.exc_info()
-                    new_exc = exc_class("Couldn't evaluate {{ %s }}: %s"
-                                        % (args, exc))
-                    raise new_exc
-            elif op == 'if':
-                expr, body = args
-                if self.evaluate(expr):
-                    self.execute(body)
-            elif op == 'for':
-                var, lis, body = args
-                vals = self.evaluate(lis)
-                for val in vals:
-                    self.context[var] = val
-                    self.execute(body)
-            else:
-                raise AssertionError("TempliteEngine doesn't grok op %r" % op)
-
-    def evaluate(self, expr):
-        """Evaluate an expression.
-
-        `expr` can have pipes and dots to indicate data access and filtering.
-
-        """
-        if "|" in expr:
-            pipes = expr.split("|")
-            value = self.evaluate(pipes[0])
-            for func in pipes[1:]:
-                value = self.evaluate(func)(value)
-        elif "." in expr:
-            dots = expr.split('.')
-            value = self.evaluate(dots[0])
-            for dot in dots[1:]:
-                try:
-                    value = getattr(value, dot)
-                except AttributeError:
-                    value = value[dot]
-                if hasattr(value, '__call__'):
-                    value = value()
-        else:
-            value = self.context[expr]
+    def do_dots(self, value, *dots):
+        """Evaluate dotted expressions at runtime."""
+        for dot in dots:
+            try:
+                value = getattr(value, dot)
+            except AttributeError:
+                value = value[dot]
+            if hasattr(value, '__call__'):
+                value = value()
         return value

python/helpers/coverage/tracer.c

@@ -0,0 +1,730 @@
+/* C-based Tracer for Coverage. */
+
+#include "Python.h"
+#include "compile.h"        /* in 2.3, this wasn't part of Python.h */
+#include "eval.h"           /* or this. */
+#include "structmember.h"
+#include "frameobject.h"
+
+/* Compile-time debugging helpers */
+#undef WHAT_LOG         /* Define to log the WHAT params in the trace function. */
+#undef TRACE_LOG        /* Define to log our bookkeeping. */
+#undef COLLECT_STATS    /* Collect counters: stats are printed when tracer is stopped. */
+
+#if COLLECT_STATS
+#define STATS(x)        x
+#else
+#define STATS(x)
+#endif
+
+/* Py 2.x and 3.x compatibility */
+
+#ifndef Py_TYPE
+#define Py_TYPE(o)    (((PyObject*)(o))->ob_type)
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+
+#define MyText_Type         PyUnicode_Type
+#define MyText_Check(o)     PyUnicode_Check(o)
+#define MyText_AS_BYTES(o)  PyUnicode_AsASCIIString(o)
+#define MyText_AS_STRING(o) PyBytes_AS_STRING(o)
+#define MyInt_FromLong(l)   PyLong_FromLong(l)
+
+#define MyType_HEAD_INIT    PyVarObject_HEAD_INIT(NULL, 0)
+
+#else
+
+#define MyText_Type         PyString_Type
+#define MyText_Check(o)     PyString_Check(o)
+#define MyText_AS_BYTES(o)  (Py_INCREF(o), o)
+#define MyText_AS_STRING(o) PyString_AS_STRING(o)
+#define MyInt_FromLong(l)   PyInt_FromLong(l)
+
+#define MyType_HEAD_INIT    PyObject_HEAD_INIT(NULL)  0,
+
+#endif /* Py3k */
+
+/* The values returned to indicate ok or error. */
+#define RET_OK      0
+#define RET_ERROR   -1
+
+/* An entry on the data stack.  For each call frame, we need to record the
+    dictionary to capture data, and the last line number executed in that
+    frame.
+*/
+typedef struct {
+    PyObject * file_data;  /* PyMem_Malloc'ed, a borrowed ref. */
+    int last_line;
+} DataStackEntry;
+
+/* The CTracer type. */
+
+typedef struct {
+    PyObject_HEAD
+
+    /* Python objects manipulated directly by the Collector class. */
+    PyObject * should_trace;
+    PyObject * warn;
+    PyObject * data;
+    PyObject * should_trace_cache;
+    PyObject * arcs;
+
+    /* Has the tracer been started? */
+    int started;
+    /* Are we tracing arcs, or just lines? */
+    int tracing_arcs;
+
+    /*
+        The data stack is a stack of dictionaries.  Each dictionary collects
+        data for a single source file.  The data stack parallels the call stack:
+        each call pushes the new frame's file data onto the data stack, and each
+        return pops file data off.
+
+        The file data is a dictionary whose form depends on the tracing options.
+        If tracing arcs, the keys are line number pairs.  If not tracing arcs,
+        the keys are line numbers.  In both cases, the value is irrelevant
+        (None).
+    */
+    /* The index of the last-used entry in data_stack. */
+    int depth;
+    /* The file data at each level, or NULL if not recording. */
+    DataStackEntry * data_stack;
+    int data_stack_alloc;       /* number of entries allocated at data_stack. */
+
+    /* The current file_data dictionary.  Borrowed. */
+    PyObject * cur_file_data;
+
+    /* The line number of the last line recorded, for tracing arcs.
+        -1 means there was no previous line, as when entering a code object.
+    */
+    int last_line;
+
+    /* The parent frame for the last exception event, to fix missing returns. */
+    PyFrameObject * last_exc_back;
+    int last_exc_firstlineno;
+
+#if COLLECT_STATS
+    struct {
+        unsigned int calls;
+        unsigned int lines;
+        unsigned int returns;
+        unsigned int exceptions;
+        unsigned int others;
+        unsigned int new_files;
+        unsigned int missed_returns;
+        unsigned int stack_reallocs;
+        unsigned int errors;
+    } stats;
+#endif /* COLLECT_STATS */
+} CTracer;
+
+#define STACK_DELTA    100
+
+static int
+CTracer_init(CTracer *self, PyObject *args_unused, PyObject *kwds_unused)
+{
+#if COLLECT_STATS
+    self->stats.calls = 0;
+    self->stats.lines = 0;
+    self->stats.returns = 0;
+    self->stats.exceptions = 0;
+    self->stats.others = 0;
+    self->stats.new_files = 0;
+    self->stats.missed_returns = 0;
+    self->stats.stack_reallocs = 0;
+    self->stats.errors = 0;
+#endif /* COLLECT_STATS */
+
+    self->should_trace = NULL;
+    self->warn = NULL;
+    self->data = NULL;
+    self->should_trace_cache = NULL;
+    self->arcs = NULL;
+
+    self->started = 0;
+    self->tracing_arcs = 0;
+
+    self->depth = -1;
+    self->data_stack = PyMem_Malloc(STACK_DELTA*sizeof(DataStackEntry));
+    if (self->data_stack == NULL) {
+        STATS( self->stats.errors++; )
+        PyErr_NoMemory();
+        return RET_ERROR;
+    }
+    self->data_stack_alloc = STACK_DELTA;
+
+    self->cur_file_data = NULL;
+    self->last_line = -1;
+
+    self->last_exc_back = NULL;
+
+    return RET_OK;
+}
+
+static void
+CTracer_dealloc(CTracer *self)
+{
+    if (self->started) {
+        PyEval_SetTrace(NULL, NULL);
+    }
+
+    Py_XDECREF(self->should_trace);
+    Py_XDECREF(self->warn);
+    Py_XDECREF(self->data);
+    Py_XDECREF(self->should_trace_cache);
+
+    PyMem_Free(self->data_stack);
+
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+#if TRACE_LOG
+static const char *
+indent(int n)
+{
+    static const char * spaces =
+        "                                                                    "
+        "                                                                    "
+        "                                                                    "
+        "                                                                    "
+        ;
+    return spaces + strlen(spaces) - n*2;
+}
+
+static int logging = 0;
+/* Set these constants to be a file substring and line number to start logging. */
+static const char * start_file = "tests/views";
+static int start_line = 27;
+
+static void
+showlog(int depth, int lineno, PyObject * filename, const char * msg)
+{
+    if (logging) {
+        printf("%s%3d ", indent(depth), depth);
+        if (lineno) {
+            printf("%4d", lineno);
+        }
+        else {
+            printf("    ");
+        }
+        if (filename) {
+            PyObject *ascii = MyText_AS_BYTES(filename);
+            printf(" %s", MyText_AS_STRING(ascii));
+            Py_DECREF(ascii);
+        }
+        if (msg) {
+            printf(" %s", msg);
+        }
+        printf("\n");
+    }
+}
+
+#define SHOWLOG(a,b,c,d)    showlog(a,b,c,d)
+#else
+#define SHOWLOG(a,b,c,d)
+#endif /* TRACE_LOG */
+
+#if WHAT_LOG
+static const char * what_sym[] = {"CALL", "EXC ", "LINE", "RET "};
+#endif
+
+/* Record a pair of integers in self->cur_file_data. */
+static int
+CTracer_record_pair(CTracer *self, int l1, int l2)
+{
+    int ret = RET_OK;
+
+    PyObject * t = Py_BuildValue("(ii)", l1, l2);
+    if (t != NULL) {
+        if (PyDict_SetItem(self->cur_file_data, t, Py_None) < 0) {
+            STATS( self->stats.errors++; )
+            ret = RET_ERROR;
+        }
+        Py_DECREF(t);
+    }
+    else {
+        STATS( self->stats.errors++; )
+        ret = RET_ERROR;
+    }
+    return ret;
+}
+
+/*
+ * The Trace Function
+ */
+static int
+CTracer_trace(CTracer *self, PyFrameObject *frame, int what, PyObject *arg_unused)
+{
+    int ret = RET_OK;
+    PyObject * filename = NULL;
+    PyObject * tracename = NULL;
+    #if WHAT_LOG || TRACE_LOG
+    PyObject * ascii = NULL;
+    #endif
+
+    #if WHAT_LOG
+    if (what <= sizeof(what_sym)/sizeof(const char *)) {
+        ascii = MyText_AS_BYTES(frame->f_code->co_filename);
+        printf("trace: %s @ %s %d\n", what_sym[what], MyText_AS_STRING(ascii), frame->f_lineno);
+        Py_DECREF(ascii);
+    }
+    #endif
+
+    #if TRACE_LOG
+    ascii = MyText_AS_BYTES(frame->f_code->co_filename);
+    if (strstr(MyText_AS_STRING(ascii), start_file) && frame->f_lineno == start_line) {
+        logging = 1;
+    }
+    Py_DECREF(ascii);
+    #endif
+
+    /* See below for details on missing-return detection. */
+    if (self->last_exc_back) {
+        if (frame == self->last_exc_back) {
+            /* Looks like someone forgot to send a return event. We'll clear
+               the exception state and do the RETURN code here.  Notice that the
+               frame we have in hand here is not the correct frame for the RETURN,
+               that frame is gone.  Our handling for RETURN doesn't need the
+               actual frame, but we do log it, so that will look a little off if
+               you're looking at the detailed log.
+
+               If someday we need to examine the frame when doing RETURN, then
+               we'll need to keep more of the missed frame's state.
+            */
+            STATS( self->stats.missed_returns++; )
+            if (self->depth >= 0) {
+                if (self->tracing_arcs && self->cur_file_data) {
+                    if (CTracer_record_pair(self, self->last_line, -self->last_exc_firstlineno) < 0) {
+                        return RET_ERROR;
+                    }
+                }
+                SHOWLOG(self->depth, frame->f_lineno, frame->f_code->co_filename, "missedreturn");
+                self->cur_file_data = self->data_stack[self->depth].file_data;
+                self->last_line = self->data_stack[self->depth].last_line;
+                self->depth--;
+            }
+        }
+        self->last_exc_back = NULL;
+    }
+
+
+    switch (what) {
+    case PyTrace_CALL:      /* 0 */
+        STATS( self->stats.calls++; )
+        /* Grow the stack. */
+        self->depth++;
+        if (self->depth >= self->data_stack_alloc) {
+            STATS( self->stats.stack_reallocs++; )
+            /* We've outgrown our data_stack array: make it bigger. */
+            int bigger = self->data_stack_alloc + STACK_DELTA;
+            DataStackEntry * bigger_data_stack = PyMem_Realloc(self->data_stack, bigger * sizeof(DataStackEntry));
+            if (bigger_data_stack == NULL) {
+                STATS( self->stats.errors++; )
+                PyErr_NoMemory();
+                self->depth--;
+                return RET_ERROR;
+            }
+            self->data_stack = bigger_data_stack;
+            self->data_stack_alloc = bigger;
+        }
+
+        /* Push the current state on the stack. */
+        self->data_stack[self->depth].file_data = self->cur_file_data;
+        self->data_stack[self->depth].last_line = self->last_line;
+
+        /* Check if we should trace this line. */
+        filename = frame->f_code->co_filename;
+        tracename = PyDict_GetItem(self->should_trace_cache, filename);
+        if (tracename == NULL) {
+            STATS( self->stats.new_files++; )
+            /* We've never considered this file before. */
+            /* Ask should_trace about it. */
+            PyObject * args = Py_BuildValue("(OO)", filename, frame);
+            tracename = PyObject_Call(self->should_trace, args, NULL);
+            Py_DECREF(args);
+            if (tracename == NULL) {
+                /* An error occurred inside should_trace. */
+                STATS( self->stats.errors++; )
+                return RET_ERROR;
+            }
+            if (PyDict_SetItem(self->should_trace_cache, filename, tracename) < 0) {
+                STATS( self->stats.errors++; )
+                return RET_ERROR;
+            }
+        }
+        else {
+            Py_INCREF(tracename);
+        }
+
+        /* If tracename is a string, then we're supposed to trace. */
+        if (MyText_Check(tracename)) {
+            PyObject * file_data = PyDict_GetItem(self->data, tracename);
+            if (file_data == NULL) {
+                file_data = PyDict_New();
+                if (file_data == NULL) {
+                    STATS( self->stats.errors++; )
+                    return RET_ERROR;
+                }
+                ret = PyDict_SetItem(self->data, tracename, file_data);
+                Py_DECREF(file_data);
+                if (ret < 0) {
+                    STATS( self->stats.errors++; )
+                    return RET_ERROR;
+                }
+            }
+            self->cur_file_data = file_data;
+            /* Make the frame right in case settrace(gettrace()) happens. */
+            Py_INCREF(self);
+            frame->f_trace = (PyObject*)self;
+            SHOWLOG(self->depth, frame->f_lineno, filename, "traced");
+        }
+        else {
+            self->cur_file_data = NULL;
+            SHOWLOG(self->depth, frame->f_lineno, filename, "skipped");
+        }
+
+        Py_DECREF(tracename);
+
+        self->last_line = -1;
+        break;
+
+    case PyTrace_RETURN:    /* 3 */
+        STATS( self->stats.returns++; )
+        /* A near-copy of this code is above in the missing-return handler. */
+        if (self->depth >= 0) {
+            if (self->tracing_arcs && self->cur_file_data) {
+                int first = frame->f_code->co_firstlineno;
+                if (CTracer_record_pair(self, self->last_line, -first) < 0) {
+                    return RET_ERROR;
+                }
+            }
+
+            SHOWLOG(self->depth, frame->f_lineno, frame->f_code->co_filename, "return");
+            self->cur_file_data = self->data_stack[self->depth].file_data;
+            self->last_line = self->data_stack[self->depth].last_line;
+            self->depth--;
+        }
+        break;
+
+    case PyTrace_LINE:      /* 2 */
+        STATS( self->stats.lines++; )
+        if (self->depth >= 0) {
+            SHOWLOG(self->depth, frame->f_lineno, frame->f_code->co_filename, "line");
+            if (self->cur_file_data) {
+                /* We're tracing in this frame: record something. */
+                if (self->tracing_arcs) {
+                    /* Tracing arcs: key is (last_line,this_line). */
+                    if (CTracer_record_pair(self, self->last_line, frame->f_lineno) < 0) {
+                        return RET_ERROR;
+                    }
+                }
+                else {
+                    /* Tracing lines: key is simply this_line. */
+                    PyObject * this_line = MyInt_FromLong(frame->f_lineno);
+                    if (this_line == NULL) {
+                        STATS( self->stats.errors++; )
+                        return RET_ERROR;
+                    }
+                    ret = PyDict_SetItem(self->cur_file_data, this_line, Py_None);
+                    Py_DECREF(this_line);
+                    if (ret < 0) {
+                        STATS( self->stats.errors++; )
+                        return RET_ERROR;
+                    }
+                }
+            }
+            self->last_line = frame->f_lineno;
+        }
+        break;
+
+    case PyTrace_EXCEPTION:
+        /* Some code (Python 2.3, and pyexpat anywhere) fires an exception event
+           without a return event.  To detect that, we'll keep a copy of the
+           parent frame for an exception event.  If the next event is in that
+           frame, then we must have returned without a return event.  We can
+           synthesize the missing event then.
+
+           Python itself fixed this problem in 2.4.  Pyexpat still has the bug.
+           I've reported the problem with pyexpat as http://bugs.python.org/issue6359 .
+           If it gets fixed, this code should still work properly.  Maybe some day
+           the bug will be fixed everywhere coverage.py is supported, and we can
+           remove this missing-return detection.
+
+           More about this fix: http://nedbatchelder.com/blog/200907/a_nasty_little_bug.html
+        */
+        STATS( self->stats.exceptions++; )
+        self->last_exc_back = frame->f_back;
+        self->last_exc_firstlineno = frame->f_code->co_firstlineno;
+        break;
+
+    default:
+        STATS( self->stats.others++; )
+        break;
+    }
+
+    return RET_OK;
+}
+
+/*
+ * Python has two ways to set the trace function: sys.settrace(fn), which
+ * takes a Python callable, and PyEval_SetTrace(func, obj), which takes
+ * a C function and a Python object.  The way these work together is that
+ * sys.settrace(pyfn) calls PyEval_SetTrace(builtin_func, pyfn), using the
+ * Python callable as the object in PyEval_SetTrace.  So sys.gettrace()
+ * simply returns the Python object used as the second argument to
+ * PyEval_SetTrace.  So sys.gettrace() will return our self parameter, which
+ * means it must be callable to be used in sys.settrace().
+ *
+ * So we make our self callable, equivalent to invoking our trace function.
+ *
+ * To help with the process of replaying stored frames, this function has an
+ * optional keyword argument:
+ *
+ *      def CTracer_call(frame, event, arg, lineno=0)
+ *
+ * If provided, the lineno argument is used as the line number, and the
+ * frame's f_lineno member is ignored.
+ */
+static PyObject *
+CTracer_call(CTracer *self, PyObject *args, PyObject *kwds)
+{
+    PyFrameObject *frame;
+    PyObject *what_str;
+    PyObject *arg;
+    int lineno = 0;
+    int what;
+    int orig_lineno;
+    PyObject *ret = NULL;
+
+    static char *what_names[] = {
+        "call", "exception", "line", "return",
+        "c_call", "c_exception", "c_return",
+        NULL
+        };
+
+    #if WHAT_LOG
+    printf("pytrace\n");
+    #endif
+
+    static char *kwlist[] = {"frame", "event", "arg", "lineno", NULL};
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!O|i:Tracer_call", kwlist,
+            &PyFrame_Type, &frame, &MyText_Type, &what_str, &arg, &lineno)) {
+        goto done;
+    }
+
+    /* In Python, the what argument is a string, we need to find an int
+       for the C function. */
+    for (what = 0; what_names[what]; what++) {
+        PyObject *ascii = MyText_AS_BYTES(what_str);
+        int should_break = !strcmp(MyText_AS_STRING(ascii), what_names[what]);
+        Py_DECREF(ascii);
+        if (should_break) {
+            break;
+        }
+    }
+
+    /* Save off the frame's lineno, and use the forced one, if provided. */
+    orig_lineno = frame->f_lineno;
+    if (lineno > 0) {
+        frame->f_lineno = lineno;
+    }
+
+    /* Invoke the C function, and return ourselves. */
+    if (CTracer_trace(self, frame, what, arg) == RET_OK) {
+        Py_INCREF(self);
+        ret = (PyObject *)self;
+    }
+
+    /* Clean up. */
+    frame->f_lineno = orig_lineno;
+
+done:
+    return ret;
+}
+
+static PyObject *
+CTracer_start(CTracer *self, PyObject *args_unused)
+{
+    PyEval_SetTrace((Py_tracefunc)CTracer_trace, (PyObject*)self);
+    self->started = 1;
+    self->tracing_arcs = self->arcs && PyObject_IsTrue(self->arcs);
+    self->last_line = -1;
+
+    /* start() returns a trace function usable with sys.settrace() */
+    Py_INCREF(self);
+    return (PyObject *)self;
+}
+
+static PyObject *
+CTracer_stop(CTracer *self, PyObject *args_unused)
+{
+    if (self->started) {
+        PyEval_SetTrace(NULL, NULL);
+        self->started = 0;
+    }
+
+    return Py_BuildValue("");
+}
+
+static PyObject *
+CTracer_get_stats(CTracer *self)
+{
+#if COLLECT_STATS
+    return Py_BuildValue(
+        "{sI,sI,sI,sI,sI,sI,sI,sI,si,sI}",
+        "calls", self->stats.calls,
+        "lines", self->stats.lines,
+        "returns", self->stats.returns,
+        "exceptions", self->stats.exceptions,
+        "others", self->stats.others,
+        "new_files", self->stats.new_files,
+        "missed_returns", self->stats.missed_returns,
+        "stack_reallocs", self->stats.stack_reallocs,
+        "stack_alloc", self->data_stack_alloc,
+        "errors", self->stats.errors
+        );
+#else
+    return Py_BuildValue("");
+#endif /* COLLECT_STATS */
+}
+
+static PyMemberDef
+CTracer_members[] = {
+    { "should_trace",       T_OBJECT, offsetof(CTracer, should_trace), 0,
+            PyDoc_STR("Function indicating whether to trace a file.") },
+
+    { "warn",               T_OBJECT, offsetof(CTracer, warn), 0,
+            PyDoc_STR("Function for issuing warnings.") },
+
+    { "data",               T_OBJECT, offsetof(CTracer, data), 0,
+            PyDoc_STR("The raw dictionary of trace data.") },
+
+    { "should_trace_cache", T_OBJECT, offsetof(CTracer, should_trace_cache), 0,
+            PyDoc_STR("Dictionary caching should_trace results.") },
+
+    { "arcs",               T_OBJECT, offsetof(CTracer, arcs), 0,
+            PyDoc_STR("Should we trace arcs, or just lines?") },
+
+    { NULL }
+};
+
+static PyMethodDef
+CTracer_methods[] = {
+    { "start",      (PyCFunction) CTracer_start,        METH_VARARGS,
+            PyDoc_STR("Start the tracer") },
+
+    { "stop",       (PyCFunction) CTracer_stop,         METH_VARARGS,
+            PyDoc_STR("Stop the tracer") },
+
+    { "get_stats",  (PyCFunction) CTracer_get_stats,    METH_VARARGS,
+            PyDoc_STR("Get statistics about the tracing") },
+
+    { NULL }
+};
+
+static PyTypeObject
+CTracerType = {
+    MyType_HEAD_INIT
+    "coverage.CTracer",        /*tp_name*/
+    sizeof(CTracer),           /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor)CTracer_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    0,                         /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    (ternaryfunc)CTracer_call, /*tp_call*/
+    0,                         /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
+    "CTracer objects",         /* tp_doc */
+    0,                         /* tp_traverse */
+    0,                         /* tp_clear */
+    0,                         /* tp_richcompare */
+    0,                         /* tp_weaklistoffset */
+    0,                         /* tp_iter */
+    0,                         /* tp_iternext */
+    CTracer_methods,           /* tp_methods */
+    CTracer_members,           /* tp_members */
+    0,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    (initproc)CTracer_init,    /* tp_init */
+    0,                         /* tp_alloc */
+    0,                         /* tp_new */
+};
+
+/* Module definition */
+
+#define MODULE_DOC PyDoc_STR("Fast coverage tracer.")
+
+#if PY_MAJOR_VERSION >= 3
+
+static PyModuleDef
+moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "coverage.tracer",
+    MODULE_DOC,
+    -1,
+    NULL,       /* methods */
+    NULL,
+    NULL,       /* traverse */
+    NULL,       /* clear */
+    NULL
+};
+
+
+PyObject *
+PyInit_tracer(void)
+{
+    PyObject * mod = PyModule_Create(&moduledef);
+    if (mod == NULL) {
+        return NULL;
+    }
+
+    CTracerType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&CTracerType) < 0) {
+        Py_DECREF(mod);
+        return NULL;
+    }
+
+    Py_INCREF(&CTracerType);
+    PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType);
+
+    return mod;
+}
+
+#else
+
+void
+inittracer(void)
+{
+    PyObject * mod;
+
+    mod = Py_InitModule3("coverage.tracer", NULL, MODULE_DOC);
+    if (mod == NULL) {
+        return;
+    }
+
+    CTracerType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&CTracerType) < 0) {
+        return;
+    }
+
+    Py_INCREF(&CTracerType);
+    PyModule_AddObject(mod, "CTracer", (PyObject *)&CTracerType);
+}
+
+#endif /* Py3k */

python/helpers/coverage/version.py

@@ -0,0 +1,9 @@
+"""The version and URL for coverage.py"""
+# This file is exec'ed in setup.py, don't import anything!
+
+__version__ = "3.7.1"                   # see detailed history in CHANGES.txt
+
+__url__ = "http://nedbatchelder.com/code/coverage"
+if max(__version__).isalpha():
+    # For pre-releases, use a version-specific URL.
+    __url__ += "/" + __version__

python/helpers/coverage/xmlreport.py

@@ -1,12 +1,14 @@
 """XML reporting for coverage.py"""
 
-import os, sys, time
+import os
+import sys
+import time
 import xml.dom.minidom
-
 from coverage import __url__, __version__
-from coverage.backward import sorted            # pylint: disable=W0622
+from coverage.backward import sorted, rpartition    # pylint: disable=W0622
 from coverage.report import Reporter
 
+
 def rate(hit, num):
     """Return the fraction of `hit`/`num`, as a string."""
     return "%.4g" % (float(hit) / (num or 1.0))
@@ -15,20 +17,19 @@ def rate(hit, num):
 class XmlReporter(Reporter):
     """A reporter for writing Cobertura-style XML coverage results."""
 
-    def __init__(self, coverage, ignore_errors=False):
-        super(XmlReporter, self).__init__(coverage, ignore_errors)
+    def __init__(self, coverage, config):
+        super(XmlReporter, self).__init__(coverage, config)
 
         self.packages = None
         self.xml_out = None
         self.arcs = coverage.data.has_arcs()
 
-    def report(self, morfs, outfile=None, config=None):
+    def report(self, morfs, outfile=None):
         """Generate a Cobertura-compatible XML report for `morfs`.
 
         `morfs` is a list of modules or filenames.
 
-        `outfile` is a file object to write the XML to.  `config` is a
-        CoverageConfig instance.
+        `outfile` is a file object to write the XML to.
 
         """
         # Initial setup.
@@ -54,7 +55,7 @@ class XmlReporter(Reporter):
 
         # Call xml_file for each file in the data.
         self.packages = {}
-        self.report_files(self.xml_file, morfs, config)
+        self.report_files(self.xml_file, morfs)
 
         lnum_tot, lhits_tot = 0, 0
         bnum_tot, bhits_tot = 0, 0
@@ -85,14 +86,23 @@ class XmlReporter(Reporter):
         # Use the DOM to write the output file.
         outfile.write(self.xml_out.toprettyxml())
 
+        # Return the total percentage.
+        denom = lnum_tot + bnum_tot
+        if denom == 0:
+            pct = 0.0
+        else:
+            pct = 100.0 * (lhits_tot + bhits_tot) / denom
+        return pct
+
     def xml_file(self, cu, analysis):
         """Add to the XML report for a single file."""
 
         # Create the 'lines' and 'package' XML elements, which
         # are populated later.  Note that a package == a directory.
-        dirname, fname = os.path.split(cu.name)
-        dirname = dirname or '.'
-        package = self.packages.setdefault(dirname, [ {}, 0, 0, 0, 0 ])
+        package_name = rpartition(cu.name, ".")[0]
+        className = cu.name
+
+        package = self.packages.setdefault(package_name, [{}, 0, 0, 0, 0])
 
         xclass = self.xml_out.createElement("class")
 
@@ -100,22 +110,22 @@ class XmlReporter(Reporter):
 
         xlines = self.xml_out.createElement("lines")
         xclass.appendChild(xlines)
-        className = fname.replace('.', '_')
+
         xclass.setAttribute("name", className)
-        ext = os.path.splitext(cu.filename)[1]
-        xclass.setAttribute("filename", cu.name + ext)
+        filename = cu.file_locator.relative_filename(cu.filename)
+        xclass.setAttribute("filename", filename.replace("\\", "/"))
         xclass.setAttribute("complexity", "0")
 
         branch_stats = analysis.branch_stats()
 
         # For each statement, create an XML 'line' element.
-        for line in analysis.statements:
+        for line in sorted(analysis.statements):
             xline = self.xml_out.createElement("line")
             xline.setAttribute("number", str(line))
 
             # Q: can we get info about the number of times a statement is
             # executed?  If so, that should be recorded here.
-            xline.setAttribute("hits", str(int(not line in analysis.missing)))
+            xline.setAttribute("hits", str(int(line not in analysis.missing)))
 
             if self.arcs:
                 if line in branch_stats: