make_patch.py

Go to the documentation of this file.

#!/usr/bin/env python3

 
import logging
import os
import re
import sys
from fnmatch import fnmatch
from subprocess import PIPE, Popen
 
 
def command(cmd, *args, **kwargs):
    """
    Simple wrapper to execute a command and return standard output and standard error.
    """
    d = {"stdout": PIPE, "stderr": PIPE}
    d.update(kwargs)
    cmd = [cmd] + list(args)
    logging.debug("Execute command: %r %r", " ".join(cmd), kwargs)
    proc = Popen(cmd, **d)
    return proc.communicate()
 
 
def cmt(*args, **kwargs):
    return command("cmt", *args, **kwargs)
 
 
def cvs(*args, **kwargs):
    return command("cvs", *args, **kwargs)
 
 
def svn(*args, **kwargs):
    return command("svn", *args, **kwargs)
 
 
def broadcast_packages():
    """
    Find the local packages the current one depends on (using 'cmt broadcast').
    Returns a list of pairs ("package name","path to the cmt directory").
    """
    # make cmt print one line per package with python syntax
    if not sys.platform.startswith("win"):
        pkg_dirs = (
            "[\n" + cmt("broadcast", r'echo "(\"<package>\", \"$PWD\"),"')[0] + "]"
        )
    else:
        pkg_dirs = (
            "[\n"
            + cmt("broadcast", r'echo ("<package>", r"%<package>root%\cmt"),')[0]
            + "]"
        )
    # Clean up a bit the output (actually needed only on Windows because of the newlines)
    pkg_dirs = "\n".join(
        [l.strip() for l in pkg_dirs.splitlines() if not l.startswith("#")]
    )
    return eval(pkg_dirs)
 
 
def matches(filename, patterns):
    """
    Returns True if any of the specified glob patterns (list of strings) matched
    the string 'filename'.
    """
    for p in patterns:
        if fnmatch(filename, p):
            logging.debug("Excluding file: %r", filename)
            return True
    return False
 
 
def expand_dirs(files, basepath=""):
    """
    Replace the entries in files that correspond to directories with the list of
    files in those directories.
    """
    if basepath:
        lb = len(basepath) + 1
    else:
        lb = 0
    newlist = []
    for f in files:
        base = os.path.join(basepath, f)
        if os.path.isdir(base):
            for root, ds, fs in os.walk(base):
                for ff in fs:
                    newlist.append(os.path.join(root, ff)[lb:])
        else:
            newlist.append(f)
    return newlist
 
 
def revision_diff_cmd(cwd):
    if os.path.isdir(os.path.join(cwd, "CVS")):
        return cvs("diff", "-upN", cwd=cwd)
    else:
        # special treatment to show new files in a way compatible with CVS
        out, err = svn("status", cwd=cwd)
 
        newfiles = [l for l in out.splitlines() if l.startswith("? ")]
        out, err = svn("diff", cwd=cwd)
        if newfiles:
            out = "\n".join(newfiles) + "\n" + out
        return out, err
 
 
def diff_pkg(name, cmtdir, exclusions=[]):
    """
    Return the patch data for a package.
    """
    rootdir = os.path.dirname(cmtdir)
    out, err = revision_diff_cmd(cwd=rootdir)
    # extract new files
    new_files = [l.split(None, 1)[1] for l in out.splitlines() if l.startswith("? ")]
    new_files = expand_dirs(new_files, rootdir)
    new_files = [f for f in new_files if not matches(f, exclusions)]
    # make diff segments for added files
    for f in new_files:
        logging.info("Added file %r", f)
        # out += "diff -u -p -N %s\n" % os.path.basename(f)
        # out += command("diff", "-upN", "/dev/null", f,
        #               cwd = rootdir)[0]
        out += f"Index: {f}\n"
        out += "===================================================================\n"
        out += command("diff", "-upN", "/dev/null", f, cwd=rootdir)[0]
    # extract removed files
    removed_files = [l.split()[-1] for l in err.splitlines() if "cannot find" in l]
    removed_files = [f for f in removed_files if not matches(f, exclusions)]
    # make diff segments for removed files (more tricky)
    for f in removed_files:
        logging.info("Removed file %r", f)
        # retrieve the original content from CVS
        orig = cvs("up", "-p", f, cwd=rootdir)[0]
        out += f"diff -u -p -N {os.path.basename(f)}\n"
        out += f"--- {f}\t1 Jan 1970 00:00:00 -0000\n"
        out += "+++ /dev/null\t1 Jan 1970 00:00:00 -0000\n"
        lines = orig.splitlines()
        out += f"@@ -1,{len(lines)} +0,0 @@\n"
        for l in lines:
            out += f"-{l}\n"
    # Fix the paths to have the package names
    rex = re.compile(r"^(Index: |\? |\+\+\+ |--- (?!/dev/null))", re.MULTILINE)
    out = rex.sub(rf"\1{name}/", out)
    return out
 
 
def main():
    from optparse import OptionParser
 
    parser = OptionParser(
        description="Produce a patch file from a CMT project. "
        "The patch contains the changes with respect "
        "to the CVS repository, including new files "
        "that are present only locally. Run the script "
        "from the cmt directory of a package."
    )
    parser.add_option(
        "-x",
        "--exclude",
        action="append",
        type="string",
        metavar="PATTERN",
        dest="exclusions",
        help="Pattern to exclude new files from the patch",
    )
    parser.add_option(
        "-o",
        "--output",
        action="store",
        type="string",
        help="Name of the file to send the output to. Standard "
        "output is used if not specified",
    )
    parser.add_option(
        "-v",
        "--verbose",
        action="store_true",
        help="Print some progress information on standard error",
    )
    parser.add_option(
        "--debug", action="store_true", help="Print debug information on standard error"
    )
    parser.set_defaults(exclusions=[])
 
    opts, args = parser.parse_args()
 
    if opts.debug:
        logging.basicConfig(level=logging.DEBUG)
    elif opts.verbose:
        logging.basicConfig(level=logging.INFO)
 
    # default exclusions
    opts.exclusions += [
        "*.py[co]",
        "*.patch",
        "cmt/cleanup.*",
        "cmt/setup.*",
        "cmt/*.make",
        "cmt/Makefile",
        "cmt/*.nmake",
        "cmt/*.nmakesav",
        "cmt/NMake",
        "cmt/install*.history",
        "cmt/build.*.log",
        "cmt/version.cmt",
        "genConf",
        "slc3_ia32_gcc323*",
        "slc4_ia32_gcc34*",
        "slc4_amd64_gcc34*",
        "slc4_amd64_gcc43*",
        "win32_vc71*",
        "i686-slc[34567]-[ig]cc*",
        "i686-slc[34567]-clang*",
        "x86_64-slc[34567]-[ig]cc*",
        "x86_64-slc[34567]-clang*",
        ".eclipse",
    ]
    if "CMTCONFIG" in os.environ:
        opts.exclusions.append(os.environ["CMTCONFIG"])
 
    # check if we are in the cmt directory before broadcasting
    if not (os.path.basename(os.getcwd()) == "cmt" and os.path.exists("requirements")):
        logging.error(
            "This script must be executed from the cmt directory of a package."
        )
        return 1
 
    pkgs = broadcast_packages()
    num_pkgs = len(pkgs)
    count = 0
 
    patch = ""
    for name, path in pkgs:
        count += 1
        logging.info(
            "Processing %s from %s (%d/%d)",
            name,
            os.path.dirname(path),
            count,
            num_pkgs,
        )
        patch += diff_pkg(name, path, opts.exclusions)
 
    if sys.platform.startswith("win"):
        # fix newlines chars
        patch = patch.replace("\r", "")
 
    if opts.output:
        logging.info("Writing patch file %r", opts.output)
        open(opts.output, "w").write(patch)
    else:
        sys.stdout.write(patch)
    return 0
 
 
if __name__ == "__main__":
    sys.exit(main())