make_patch.py

Go to the documentation of this file.

#!/usr/bin/env python

import os
import sys
import re
from subprocess import Popen, PIPE, STDOUT
from fnmatch import fnmatch
import logging


def command(cmd, *args, **kwargs):
    """
    Simple wrapper to execute a command and return standard output and standard error.
    """
    d = {"stdout": PIPE, "stderr": PIPE}
    d.update(kwargs)
    cmd = [cmd] + list(args)
    logging.debug("Execute command: %r %r", " ".join(cmd), kwargs)
    proc = apply(Popen, (cmd, ), d)
    return proc.communicate()


cmt = lambda *args, **kwargs: apply(command, ("cmt", ) + args, kwargs)
cvs = lambda *args, **kwargs: apply(command, ("cvs", ) + args, kwargs)
svn = lambda *args, **kwargs: apply(command, ("svn", ) + args, kwargs)


def broadcast_packages():
    """
    Find the local packages the current one depends on (using 'cmt broadcast').
    Returns a list of pairs ("package name","path to the cmt directory").
    """
    # make cmt print one line per package with python syntax
    if not sys.platform.startswith("win"):
        pkg_dirs = "[\n" + cmt("broadcast",
                               r'echo "(\"<package>\", \"$PWD\"),"')[0] + ']'
    else:
        pkg_dirs = "[\n" + cmt(
            "broadcast",
            r'echo ("<package>", r"%<package>root%\cmt"),')[0] + ']'
    # Clean up a bit the output (actually needed only on Windows because of the newlines)
    pkg_dirs = "\n".join(
        [l.strip() for l in pkg_dirs.splitlines() if not l.startswith("#")])
    return eval(pkg_dirs)


def matches(filename, patterns):
    """
    Returns True if any of the specified glob patterns (list of strings) matched
    the string 'filename'.
    """
    for p in patterns:
        if fnmatch(filename, p):
            logging.debug("Excluding file: %r", filename)
            return True
    return False


def expand_dirs(files, basepath=""):
    """
    Replace the entries in files that correspond to directories with the list of
    files in those directories.
    """
    if basepath:
        lb = len(basepath) + 1
    else:
        lb = 0
    newlist = []
    for f in files:
        base = os.path.join(basepath, f)
        if os.path.isdir(base):
            for root, ds, fs in os.walk(base):
                for ff in fs:
                    newlist.append(os.path.join(root, ff)[lb:])
        else:
            newlist.append(f)
    return newlist


def revision_diff_cmd(cwd):
    if os.path.isdir(os.path.join(cwd, "CVS")):
        return cvs("diff", "-upN", cwd=cwd)
    else:
        # special treatment to show new files in a way compatible with CVS
        out, err = svn("status", cwd=cwd)

        newfiles = [l for l in out.splitlines() if l.startswith("? ")]
        out, err = svn("diff", cwd=cwd)
        if newfiles:
            out = "\n".join(newfiles) + "\n" + out
        return out, err


def diff_pkg(name, cmtdir, exclusions=[]):
    """
    Return the patch data for a package.
    """
    rootdir = os.path.dirname(cmtdir)
    out, err = revision_diff_cmd(cwd=rootdir)
    # extract new files
    new_files = [
        l.split(None, 1)[1] for l in out.splitlines() if l.startswith("? ")
    ]
    new_files = expand_dirs(new_files, rootdir)
    new_files = [f for f in new_files if not matches(f, exclusions)]
    # make diff segments for added files
    for f in new_files:
        logging.info("Added file %r", f)
        #out += "diff -u -p -N %s\n" % os.path.basename(f)
        # out += command("diff", "-upN", "/dev/null", f,
        #               cwd = rootdir)[0]
        out += "Index: %s\n" % f
        out += "===================================================================\n"
        out += command("diff", "-upN", "/dev/null", f, cwd=rootdir)[0]
    # extract removed files
    removed_files = [
        l.split()[-1] for l in err.splitlines() if "cannot find" in l
    ]
    removed_files = [f for f in removed_files if not matches(f, exclusions)]
    # make diff segments for removed files (more tricky)
    for f in removed_files:
        logging.info("Removed file %r", f)
        # retrieve the original content from CVS
        orig = cvs("up", "-p", f, cwd=rootdir)[0]
        out += "diff -u -p -N %s\n" % os.path.basename(f)
        out += "--- %s\t1 Jan 1970 00:00:00 -0000\n" % f
        out += "+++ /dev/null\t1 Jan 1970 00:00:00 -0000\n"
        lines = orig.splitlines()
        out += "@@ -1,%d +0,0 @@\n" % len(lines)
        for l in lines:
            out += '-%s\n' % l
    # Fix the paths to have the package names
    rex = re.compile(r"^(Index: |\? |\+\+\+ |--- (?!/dev/null))", re.MULTILINE)
    out = rex.sub(r"\1%s/" % name, out)
    return out


def main():
    from optparse import OptionParser
    parser = OptionParser(
        description="Produce a patch file from a CMT project. "
        "The patch contains the changes with respect "
        "to the CVS repository, including new files "
        "that are present only locally. Run the script "
        "from the cmt directory of a package.")
    parser.add_option(
        "-x",
        "--exclude",
        action="append",
        type="string",
        metavar="PATTERN",
        dest="exclusions",
        help="Pattern to exclude new files from the patch")
    parser.add_option(
        "-o",
        "--output",
        action="store",
        type="string",
        help="Name of the file to send the output to. Standard "
        "output is used if not specified")
    parser.add_option(
        "-v",
        "--verbose",
        action="store_true",
        help="Print some progress information on standard error")
    parser.add_option(
        "--debug",
        action="store_true",
        help="Print debug information on standard error")
    parser.set_defaults(exclusions=[])

    opts, args = parser.parse_args()

    if opts.debug:
        logging.basicConfig(level=logging.DEBUG)
    elif opts.verbose:
        logging.basicConfig(level=logging.INFO)

    # default exclusions
    opts.exclusions += [
        "*.py[co]",
        "*.patch",
        "cmt/cleanup.*",
        "cmt/setup.*",
        "cmt/*.make",
        "cmt/Makefile",
        "cmt/*.nmake",
        "cmt/*.nmakesav",
        "cmt/NMake",
        "cmt/install*.history",
        "cmt/build.*.log",
        "cmt/version.cmt",
        "genConf",
        "slc3_ia32_gcc323*",
        "slc4_ia32_gcc34*",
        "slc4_amd64_gcc34*",
        "slc4_amd64_gcc43*",
        "win32_vc71*",
        "i686-slc[34567]-[ig]cc*",
        "i686-slc[34567]-clang*",
        "x86_64-slc[34567]-[ig]cc*",
        "x86_64-slc[34567]-clang*",
        ".eclipse",
    ]
    if "CMTCONFIG" in os.environ:
        opts.exclusions.append(os.environ["CMTCONFIG"])

    # check if we are in the cmt directory before broadcasting
    if not (os.path.basename(os.getcwd()) == "cmt"
            and os.path.exists("requirements")):
        logging.error(
            "This script must be executed from the cmt directory of a package."
        )
        return 1

    pkgs = broadcast_packages()
    num_pkgs = len(pkgs)
    count = 0

    patch = ""
    for name, path in pkgs:
        count += 1
        logging.info("Processing %s from %s (%d/%d)", name,
                     os.path.dirname(path), count, num_pkgs)
        patch += diff_pkg(name, path, opts.exclusions)

    if sys.platform.startswith("win"):
        # fix newlines chars
        patch = patch.replace("\r", "")

    if opts.output:
        logging.info("Writing patch file %r", opts.output)
        open(opts.output, "w").write(patch)
    else:
        sys.stdout.write(patch)
    return 0


if __name__ == "__main__":
    sys.exit(main())