collect_for_ctest.py

Go to the documentation of this file.

"""
pytest plugin that report collected pytest files as CTest tests
 
This plugin is not meant to be used directly, but it is invoked by the
CMake function `gaudi_add_pytest()`
"""
 
import os
import re
from collections import defaultdict
from pathlib import Path
 
 
def pytest_addoption(parser, pluginmanager):
    parser.addoption(
        "--ctest-output-file",
        help="name of the file to write to communicate to ctest the discovered tests",
    )
    parser.addoption(
        "--ctest-pytest-command",
        default="pytest",
        help="how pytest has to be invoked (e.g. using wrapper commands)",
    )
    parser.addoption(
        "--ctest-pytest-root-dir",
        default=Path.cwd(),
        help="root directory to compute test names",
    )
    parser.addoption(
        "--ctest-prefix",
        default="",
        help="string to prefix to the generated test names",
    )
    parser.addoption(
        "--ctest-label",
        default=["pytest"],
        action="append",
        help="labels to attach to the test (the label pytest is always added)",
    )
    parser.addoption(
        "--ctest-properties",
        default=[],
        action="append",
        help="test properties to set for all discovered tests",
    )
    parser.addoption(
        "--ctest-binary-dir",
        default=None,
        help="value of CMAKE_CURRENT_BINARY_DIR from which gaudi_add_pytest was invoked",
    )
    parser.addoption(
        "--ctest-coverage",
        default="",
        help="select modules for which produce coverage reports",
    )
    parser.addoption(
        "--ctest-coverage-command",
        default="coverage report",
        help="how coverage should be invoked to produce the final report",
    )
 
 
def pytest_collectstart(collector):
    session, config = collector.session, collector.config
    args = {
        name[6:]: getattr(config.option, name)
        for name in dir(config.option)
        if name.startswith("ctest_")
    }
 
    if args.get("binary_dir"):
        # $CMAKE_CURRENT_BINARY_DIR will be set by CTest when the test is run
        # so, for consistency, we set it at collection time too
        os.environ["CMAKE_CURRENT_BINARY_DIR"] = args["binary_dir"]
 
    session.ctest_args = args
 
 
def pytest_collection_modifyitems(session, config, items):
    if not session.ctest_args.get("output_file"):
        # nothing to do if no output file is specified
        return
 
    session.ctest_files = set(item.path for item in items)
    session.ctest_fixture_setup = defaultdict(set)
    session.ctest_fixture_required = defaultdict(set)
    for item in items:
        for marker in ("ctest_fixture_setup", "ctest_fixture_required"):
            for mark in item.iter_markers(name=marker):
                getattr(session, marker)[item.path].update(mark.args)
 
 
TEST_DESC_TEMPLATE = """
add_test({name} {pytest_cmd} {path})
set_tests_properties({name} PROPERTIES {properties})
"""
 
 
def _resolve_sanitizer_lib(name):
    """Resolve a bare sanitizer runtime library name (e.g. "libasan.so") to
    an absolute path by searching LD_LIBRARY_PATH ourselves, instead of
    leaving it bare for the dynamic loader to search for later.
 
    PRELOAD_SANITIZER_LIB (set by the sanitizer toolchain, see
    lcg-toolchains' fragments/sanitizers/settings.cmake) is exported as a
    bare filename. That's fine here: this function runs inside the same
    `run`-wrapper environment used to launch this very pytest collection
    process, which is known to have LD_LIBRARY_PATH set up correctly (the
    sanitizer library does get found by *something* in this process, or
    ASan wouldn't be able to report "runtime does not come first in
    initial library list" at all -- it would fail to load entirely). But
    that's not guaranteed for whatever process actually *runs* each test
    later, which is a different invocation context (verified: pytest-based
    CTest tests print "ld.so: ... cannot be preloaded (cannot open shared
    object file): ignored" and silently run without the sanitizer active).
    Resolving to an absolute path now, once, and baking it into each
    test's ENVIRONMENT property removes that dependency entirely -- an
    absolute path needs no search at test-run time.
    """
    if os.path.isabs(name):
        return name
    for directory in os.environ.get("LD_LIBRARY_PATH", "").split(":"):
        if directory and os.path.isfile(os.path.join(directory, name)):
            return os.path.join(directory, name)
    return name  # not found: fall back to the bare name, as before
 
 
def pytest_collection_finish(session):
    args = session.ctest_args
    output_filename = args.get("output_file")
 
    if not output_filename:
        # nothing to do if no output file is specified
        return
 
    output = open(output_filename, "w")
    output_rootdir = Path(output_filename).parent
 
    coverage = args["coverage"].split(",") if args["coverage"] else []
    if coverage:
        args["pytest_command"] += " --cov-report= --cov-reset " + " ".join(
            f"--cov={module}" for module in coverage
        )
 
    properties = 'LABELS "{}" '.format(";".join(args["label"]))
    env_entries = []
    if args.get("binary_dir"):
        env_entries.append(f"CMAKE_CURRENT_BINARY_DIR={args['binary_dir']}")
    preload_sanitizer_lib = os.environ.get("PRELOAD_SANITIZER_LIB")
    if preload_sanitizer_lib:
        resolved = ":".join(
            _resolve_sanitizer_lib(name)
            for name in preload_sanitizer_lib.split(":")
            if name
        )
        env_entries.append(f"LD_PRELOAD={resolved}")
    if env_entries:
        properties += 'ENVIRONMENT "{}" '.format(";".join(env_entries))
    properties += " ".join(args["properties"])
 
    producers = defaultdict(list)  # test name -> list of fixtures it produces
    consumers = defaultdict(list)  # test name -> list of fixtures it depends on
    fixtures = defaultdict(list)  # fixture name -> list of tests that produce it
    names = []
    for path in sorted(session.ctest_files):
        name = os.path.relpath(path, args["pytest_root_dir"]).replace("/", ".")
        if name.endswith(".py"):
            name = name[:-3]
        if name == ".":
            # when we pass a single file to pytest, pytest_root_dir and path are the same
            # and relpath returns ".", so we take the prefix and drop the final dot
            # (see https://gitlab.cern.ch/gaudi/Gaudi/-/issues/354)
            name = args["prefix"][:-1]
        else:
            name = args["prefix"] + name
        pytest_cmd = args["pytest_command"]
 
        output.write(
            TEST_DESC_TEMPLATE.format(
                name=name,
                path=path,
                pytest_cmd=pytest_cmd,
                properties=properties,
            )
        )
 
        if session.ctest_fixture_setup.get(path):
            for fixture in session.ctest_fixture_setup[path]:
                producers[name].append(fixture)
                fixtures[fixture].append(name)
 
        if session.ctest_fixture_required.get(path):
            for fixture in session.ctest_fixture_required[path]:
                consumers[name].append(fixture)
 
        names.append(name)
 
    for name in names:
        if name in producers:
            output.write(
                'set_tests_properties('
                f'{name} PROPERTIES FIXTURES_SETUP "{";".join(producers[name])}")\n'
            )
 
        if name in consumers:
            producer_test_names = []
            for fixture in consumers[name]:
                producer_test_names.extend(fixtures[fixture])
            output.write(
                'if (DEFINED ENV{PYTEST_DISABLE_FIXTURES_REQUIRED})\n'
                f'  set_tests_properties({name} PROPERTIES DEPENDS "{";".join(producer_test_names)}")\n'
                'else()\n'
                f'  set_tests_properties({name} PROPERTIES FIXTURES_REQUIRED "{";".join(consumers[name])}")\n'
                'endif()\n'
            )
 
        # we force one test to be run one by one
        if coverage:
            # generate one coverage file per test
            output.write(
                f"set_tests_properties({name} PROPERTIES ENVIRONMENT COVERAGE_FILE={output_rootdir}/.coverage.{name})\n"
                f"set_tests_properties({name} PROPERTIES FIXTURES_SETUP {name})\n"
            )
 
    if coverage and names:
        combine_test = (args["prefix"] + "coverage_combine").replace("/", ".")
        combine_command = re.sub(
            r" report .*",
            f" combine {' '.join(f'{output_rootdir}/.coverage.{n}' for n in names)}",
            args["coverage_command"],
        )
        output.write(
            TEST_DESC_TEMPLATE.format(
                name=combine_test,
                path="",
                pytest_cmd=combine_command,
                properties=properties,
            )
        )
        output.write(
            f"set_tests_properties({combine_test} PROPERTIES ENVIRONMENT COVERAGE_FILE={output_rootdir}/.coverage)\n"
            f"set_tests_properties({combine_test} PROPERTIES FIXTURES_REQUIRED \"{';'.join(names)}\")\n"
            f"set_tests_properties({combine_test} PROPERTIES FIXTURES_SETUP {combine_test})\n"
        )
 
        name = (args["prefix"] + "coverage_report").replace("/", ".")
        output.write(
            TEST_DESC_TEMPLATE.format(
                name=name,
                path="",
                pytest_cmd=f"{args['coverage_command']}",
                properties=properties + " LABELS coverage",
            )
        )
        # coverage reports require all related tests to be run first
        output.write(
            f"set_tests_properties({name} PROPERTIES ENVIRONMENT COVERAGE_FILE={output_rootdir}/.coverage)\n"
            f"set_tests_properties({name} PROPERTIES FIXTURES_REQUIRED {combine_test})\n"
        )
 
    output.close()
 
 
def pytest_configure(config):
    config.addinivalue_line(
        "markers",
        "ctest_fixture_setup(name): mark test to set up a fixture needed by another test",
    )
    config.addinivalue_line(
        "markers",
        "ctest_fixture_required(name): mark test to require a fixture set up by another test",
    )