testsuite/python/tests/test_100_1.py

############################################################################
# Copyright (C) SchedMD LLC.
############################################################################
from pathlib import Path

import pytest

import atf

xfail_tests = []
skip_tests = []
"""
List for xfails and skips to append known issues (as code).

The xfail_tests is of tuples of:
- file name relative to testsuite_check_dir/tests
- test case name used tcase_add_test(), or None for compilation xfailures
- referenced reason of the known issue

The skip_testis list is equivalent, but without the test case name because
the whole test file won't be compiled nor run.

e.g.

To xfail specific test case in a test file:

xfail_tests.append(
   (
       "common/test_timespec_t.c",
       "test_normalize",
       "Issue #50096. Math operations fail cross negatives",
   )
)

Or to xfail the whole test file:

xfail_tests.append(
   (
       "common/test_timespec_t.c",
       None,
       "Issue #NNNN. Test don't compile due OS issue",
   )
)

Or to skip a test:

skip_tests.append(
   (
       "common/test_timespec_t.c",
       "Feature available in versions 24.11+",
   )
)

Note that xfails will run normally and when they fail we'll get the reason in
the output. If they pass we'll get notified as "failure", so we can check them
and hopefully remove them from the xfail list.
"""

# TODO: Remove xfail_tests.append() once their issue is fixed.
if atf.get_version() < (26, 11):
    skip_tests.append(
        (
            "common/test_json_patch.c",
            "serializer/json-patch plugin is new in 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_job_record.c",
            "Ticket 25566: job priority reporting tests require Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "interfaces/test_gres_tpu.c",
            "Issue 51191: gres/tpu plugin added in 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_env.c",
            "test_setenvfs_success",
            "Issue #51135: setenvfs() uses putenv() before 26.11; the interposed setenv() is never triggered",
        )
    )
    xfail_tests.append(
        (
            "common/test_env.c",
            "test_setenvfs_setenv_failure",
            "Issue #51135: setenvfs() uses putenv() before 26.11; fail_setenv has no effect so it always returns 0",
        )
    )
    xfail_tests.append(
        (
            "conmgr/test_conmgr.c",
            "test_watch_shutdown_before_start",
            "Issue #51358: watch() early shutdown neither clears watch_thread nor wakes waiters before 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_http_router.c",
            "Ticket 24549: Only test http_router starting at Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "interfaces/test_serializer.c",
            "serializer JSON compliance test introduced in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "interfaces/test_runtime.c",
            "runtime interface introduced in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_workq.c",
            "workq is new in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_workerpool.c",
            "workerpool is new in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_pack_data.c",
            "pack_data() is new in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_kv_protocol.c",
            "KV data fields are new in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_pack_job_desc.c",
            "job_desc_msg_t runtime_data is new in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_xutf.c",
            "Issue 50192: xutf UTF-8 handler introduced in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "plugins/relay_io/test_relay_io_common.c",
            "Issue 51035: relay_io introduced in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "plugins/relay_io/test_ring_buffer.c",
            "Issue 51035: relay_io introduced in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_log.c",
            "Issue #50192. LogTimeFormat options introduced in Slurm 26.11",
        )
    )
    skip_tests.append(
        (
            "common/test_http_con.c",
            "Issue #50528: conmgr/workerpool added in 26.11+",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "push_oversized_number_is_rejected",
            "A host name whose number does not fit is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "create_oversized_number_is_rejected",
            "A host name whose number does not fit is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "create_oversized_number_in_list_is_rejected",
            "A host name whose number does not fit is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "create_oversized_number_in_bracket_is_rejected",
            "A bracketed number that does not fit is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "create_oversized_range_end_is_rejected",
            "A range end that does not fit is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "find_oversized_number_is_not_found",
            "A host name whose number does not fit is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "push_ulong_max_is_rejected",
            "A host number of ULONG_MAX is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "create_range_to_ulong_max_is_rejected",
            "A host number of ULONG_MAX is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_name.c",
            "create_range_trailing_chars_is_rejected",
            "A range start with trailing characters is rejected starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_find.c",
            "find_absent_shared_prefix",
            "hostlist_find() stops matching an absent host that shares a list's prefix starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_find.c",
            "find_absent_swallowed_suffix",
            "hostlist_find() stops matching an absent host that shares a list's prefix starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_find.c",
            "find_member_among_decoys",
            "hostlist_find() stops letting a non-matching range decide a later one starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_find.c",
            "delete_absent_is_noop",
            "hostlist_delete_host() stops deleting a host that shares a list's prefix starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_find.c",
            "within_multiple_hosts",
            "hostset_within() stops matching an absent host that shares a list's prefix starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_find.c",
            "find_bracket_suffix",
            "hostlist_find() returns the right index for a bracket with a suffix starting at 26.11",
        )
    )
    xfail_tests.append(
        (
            "common/test_hostlist_find.c",
            "find_bracket_suffix_delete",
            "hostlist_find() returns the right index for a bracket with a suffix starting at 26.11",
        )
    )
if atf.get_version() < (25, 5) or (26, 11) < atf.get_version()[:2]:
    skip_tests.append(
        (
            "plugins/data_parser/test_parsers_v0_0_43.c",
            "data_parser/v0.0.43 available from Slurm 25.05 to 26.11",
        )
    )
if atf.get_version() < (25, 11) or (27, 5) < atf.get_version()[:2]:
    skip_tests.append(
        (
            "plugins/data_parser/test_parsers_v0_0_44.c",
            "data_parser/v0.0.44 available from Slurm 25.11 to 27.05",
        )
    )
if atf.get_version() < (26, 5) or (27, 11) < atf.get_version()[:2]:
    skip_tests.append(
        (
            "plugins/data_parser/test_parsers_v0_0_45.c",
            "data_parser/v0.0.45 available from Slurm 26.05 to 27.11",
        )
    )
if atf.get_version() < (26, 11) or (28, 5) < atf.get_version()[:2]:
    skip_tests.append(
        (
            "plugins/data_parser/test_parsers_v0_0_46.c",
            "data_parser/v0.0.46 available from Slurm 26.11 to 28.05",
        )
    )
if atf.get_version() < (26, 5):
    xfail_tests.append(
        (
            "common/test_timespec_t.c",
            "test_rem",
            "Issue #50096. Math operations fail cross negatives",
        )
    )
    xfail_tests.append(
        (
            "common/test_uid.c",
            "test_negative_cache",
            "uid negative cache does not exist until Slurm 26.05.0",
        )
    )
    xfail_tests.append(
        (
            "common/test_uid.c",
            "test_uid_lookup_time",
            "uid_from_string caching does not exist until Slurm 26.05.0",
        )
    )
    xfail_tests.append(
        (
            "conmgr/test_polling.c",
            "test_interrupt_before_poll",
            "Ticket 25227: conmgr pollctl interrupt lost-wakeup fix lands in 26.05",
        )
    )
    xfail_tests.append(
        (
            "conmgr/test_polling.c",
            "test_interrupt_before_poll_poll",
            "Ticket 25227: conmgr pollctl interrupt lost-wakeup fix lands in 26.05",
        )
    )
    xfail_tests.append(
        (
            "conmgr/test_polling.c",
            "test_stale_revents_poll",
            "Ticket 25227: conmgr pollctl interrupt lost-wakeup fix lands in 26.05",
        )
    )
    xfail_tests.append(
        (
            "topology/test_route_tree.c",
            "test_disconnected_trunks",
            "Ticket 25473: topology/tree leaf-relay fan-out fix lands in 26.05",
        )
    )
if atf.get_version() < (25, 11):
    skip_tests.append(
        (
            "interfaces/test_data_parsers.c",
            "test_data_parsers doesn't compile for versions < 25.11",
        )
    )
    xfail_tests.append(
        (
            "conmgr/test_conmgr.c",
            "test_reinit",
            "conmgr_init() has different parameters for versions < 25.11",
        )
    )
if atf.get_version() < (25, 5):
    skip_tests.append(
        (
            "common/test_timespec_t.c",
            "timespec_t test doesn't compile for versions < 25.05",
        )
    )
    skip_tests.append(
        (
            "conmgr/test_conmgr.c",
            "conmgr test doesn't compile for versions < 25.05",
        )
    )
    skip_tests.append(
        (
            "interfaces/test_gres_node_config.c",
            "gres_get_node_conf_list() is added in 25.05+",
        )
    )

# Create a test_function() for all test file in the testsuite_check_dir.
# All the test_function()s will be run by pytest as if actually defined here.

testsuite_check_tests_dir = Path(atf.properties["testsuite_check_dir"] + "/tests")
test_files = list(testsuite_check_tests_dir.rglob("test_*.c"))

for test in test_files:
    src = str(Path(test).relative_to(testsuite_check_tests_dir))
    test_name = "test_" + src.replace(".c", "").replace("test_", "").replace(
        ".", "_"
    ).replace("-", "_").replace("/", "_")

    def make_test(src):

        def test_func():
            if atf.is_upgrade_setup():
                pytest.skip("The libcheck test doesn't work with upgrade setups")

            # Run the libcheck tests and get the xml parsed results
            test_results = atf.run_check_test(src, build_args="-Wl,--export-dynamic")

            if "test" not in test_results:
                pytest.fail(
                    f"{src}: zero tests ran; the suite's setup fixture probably failed"
                )

            # Get a list of test cases (ensure it's a list)
            test_cases = test_results["test"]
            if not isinstance(test_cases, list):
                test_cases = [test_cases]

            # Get the list of xfail for this suite (if any)
            suite_xfails = [xfail for xfail in xfail_tests if xfail[0] == src]

            # Get the failure in the suite and evaluate if they are xfail or we should fail
            suite_fails = [test for test in test_cases if test["@result"] != "success"]
            xfails_found = []
            real_fails = []
            for failure in suite_fails:
                # An xfail covers every iteration of a loop/parametric test
                xfail = next(
                    (
                        xfail
                        for xfail in suite_xfails + xfails_found
                        if xfail[1] == failure["id"]
                    ),
                    None,
                )
                if xfail:
                    # failure is an xfail
                    if xfail in suite_xfails:
                        suite_xfails.remove(xfail)
                        xfails_found.append(xfail)
                else:
                    # failure is not xfail
                    real_fails.append(
                        f"{failure['id']}[{failure['iteration']}]"
                        f" ({failure['fn']}): {failure['message']}"
                    )

            # Report every failure, not just the first one. A loop test runs
            # one unit per item, so a systemic breakage fails many of them.
            if real_fails:
                pytest.fail("\n".join(real_fails))

            # Check if we still have remaining xfails (not popped above)
            for xfail in suite_xfails:
                xpass = next(
                    (test for test in test_results["test"] if test["id"] == xfail[1]),
                    None,
                )
                if xpass:
                    pytest.fail(
                        f"{xfail[1]} is marked as xfail but {xpass['@result']}: {xpass['message']}"
                    )
                else:
                    pytest.fail(f"{xfail[1]} is marked as xfail but never run")

            # Finally, if some xfail failed, mark the test as xfail
            xfail_msg = ""
            for xfail in xfails_found:
                xfail_msg += f"{xfail[1]}: {xfail[2]}\n"
            if xfail_msg != "":
                pytest.xfail(xfail_msg)

        # If there's an xfail for the test file (test_case is None), always xfail
        test_file_xfails = [
            xfail for xfail in xfail_tests if xfail[0] == src and xfail[1] is None
        ]

        if test_file_xfails:
            test_func = pytest.mark.xfail(reason=test_file_xfails[0][2])(test_func)

        # Skip the test if listed
        test_file_skip = [skip for skip in skip_tests if skip[0] == src]

        if test_file_skip:
            test_func = pytest.mark.skip(reason=test_file_skip[0][1])(test_func)

        return test_func

    globals()[test_name] = make_test(src)