testsuite/python/tests/test_100_1.py

############################################################################
# Copyright (C) SchedMD LLC.
############################################################################
import atf
import pytest
from pathlib import Path

xfail_tests = []
skip_tests = []
"""
List for xfails and skips to append known issues (as code).

The xfail_tests is of tuples of:
- file name relative to testsuite_check_dir
- test case name used tcase_add_test(), or None for compilation xfailures
- referenced reason of the known issue

The skip_testis list is equivalent, but without the test case name because
the whole test file won't be compiled nor run.

e.g.

To xfail specific test case in a test file:

xfail_tests.append(
   (
       "common/test_timespec_t.c",
       "test_normalize",
       "Issue #50096. Math operations fail cross negatives",
   )
)

Or to xfail the whole test file:

xfail_tests.append(
   (
       "common/test_timespec_t.c",
       None,
       "Issue #NNNN. Test don't compile due OS issue",
   )
)

Or to skip a test:

skip_tests.append(
   (
       "common/test_timespec_t.c",
       "Feature available in versions 24.11+",
   )
)

Note that xfails will run normally and when they fail we'll get the reason in
the output. If they pass we'll get notified as "failure", so we can check them
and hopefully remove them from the xfail list.
"""

# TODO: Remove xfail_tests.append() once their issue is fixed.
if atf.get_version() < (25, 11):
    skip_tests.append(
        (
            "interfaces/test_data_parsers.c",
            "test_data_parsers doesn't compile for versions < 25.11",
        )
    )
if atf.get_version() < (25, 5):
    skip_tests.append(
        (
            "common/test_timespec_t.c",
            "timespec_t test doesn't compile for versions < 25.05",
        )
    )
    skip_tests.append(
        (
            "common/test_conmgr.c",
            "conmgr test doesn't compile for versions < 25.05",
        )
    )
else:
    xfail_tests.append(
        (
            "common/test_timespec_t.c",
            "test_normalize",
            "Issue #50096. Math operations fail cross negatives",
        )
    )

# Create a test_function() for all test file in the testsuite_check_dir.
# All the test_function()s will be run by pytest as if actually defined here.

testsuite_check_dir = Path(atf.properties["testsuite_check_dir"])
test_files = list(testsuite_check_dir.rglob("test_*.c"))

for test in test_files:
    src = str(Path(test).relative_to(testsuite_check_dir))
    test_name = "test_" + src.replace(".c", "").replace("test_", "").replace(
        ".", "_"
    ).replace("-", "_").replace("/", "_")

    def make_test(src):

        def test_func():
            if atf.is_upgrade_setup():
                pytest.skip("The libcheck test doesn't work with upgrade setups")

            # Run the libcheck tests and get the xml parsed results
            test_results = atf.run_check_test(src)

            # Get a list of test cases (ensure it's a list)
            test_cases = test_results["test"]
            if not isinstance(test_cases, list):
                test_cases = [test_cases]

            # Get the list of xfail for this suite (if any)
            suite_xfails = [xfail for xfail in xfail_tests if xfail[0] == src]

            # Get the failure in the suite and evaluate if they are xfail or we should fail
            suite_fails = [test for test in test_cases if test["@result"] != "success"]
            xfails_found = []
            for failure in suite_fails:
                for i, xfail in enumerate(suite_xfails):
                    if xfail[1] == failure["id"]:
                        # failure is an xfail
                        suite_xfails.pop(i)
                        xfails_found.append(xfail)
                        break
                else:
                    # failure is not xfail
                    pytest.fail(f"{failure['id']}: {failure['message']}")

            # Check if we still have remaining xfails (not popped above)
            for xfail in suite_xfails:
                xpass = next(
                    (test for test in test_results["test"] if test["id"] == xfail[1]),
                    None,
                )
                if xpass:
                    pytest.fail(
                        f"{xfail[1]} is marked as xfail but {xpass['@result']}: {xpass['message']}"
                    )
                else:
                    pytest.fail(f"{xfail[1]} is marked as xfail but never run")

            # Finally, if some xfail failed, mark the test as xfail
            xfail_msg = ""
            for xfail in xfails_found:
                xfail_msg += f"{xfail[1]}: {xfail[2]}\n"
            if xfail_msg != "":
                pytest.xfail(xfail_msg)

        # If there's an xfail for the test file (test_case is None), always xfail
        test_file_xfails = [
            xfail for xfail in xfail_tests if xfail[0] == src and xfail[1] is None
        ]

        if test_file_xfails:
            test_func = pytest.mark.xfail(reason=test_file_xfails[0][2])(test_func)

        # Skip the test if listed
        test_file_skip = [skip for skip in skip_tests if skip[0] == src]

        if test_file_skip:
            test_func = pytest.mark.skip(reason=test_file_skip[0][1])(test_func)

        return test_func

    globals()[test_name] = make_test(src)