| ############################################################################ |
| # Copyright (C) SchedMD LLC. |
| ############################################################################ |
| import atf |
| import datetime |
| import os |
| import pytest |
| import re |
| |
| # Period Boundaries |
| # Mon Dec 31 23:00:00 2007 |
| period_start_datetime = datetime.datetime(2007, 12, 31, 23, 0, 0) |
| period_start_epoch = int(period_start_datetime.timestamp()) |
| period_start_string = period_start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| # Thu Jan 31 23:59:59 2008 |
| period_end_datetime = datetime.datetime(2008, 1, 31, 23, 59, 59) |
| period_end_epoch = int(period_end_datetime.timestamp()) |
| period_end_string = period_end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| # Midnight Fri Thu Jan 31 00:00:00 2008 |
| midnight_datetime = datetime.datetime(2008, 1, 31, 0, 0, 0) |
| midnight_epoch = int(midnight_datetime.timestamp()) |
| |
| # Identities |
| uid = os.geteuid() |
| gid = os.getegid() |
| |
| # Clusters |
| cluster1 = "cluster1" |
| cluster2 = "cluster2" |
| |
| # Accounts |
| account1 = "account1" |
| account2 = "account2" |
| account3 = "account3" |
| |
| # Workload Characterization Keys |
| wckey1 = "wckey1" |
| |
| # Users |
| user1 = "user1" |
| user2 = "user2" |
| |
| # Nodes |
| node0 = f"{cluster1}_node0" |
| node1 = f"{cluster1}_node1" |
| node_list = f"{cluster1}_node[0-1]" |
| node0_cpus = 2 |
| node1_cpus = 2 |
| cluster_cpus = node0_cpus + node1_cpus |
| |
| # node0 down |
| node0_down_start_epoch = period_start_epoch + (45 * 60) |
| node0_down_end_epoch = period_start_epoch + (75 * 60) |
| node0_start_datetime = datetime.datetime.fromtimestamp(node0_down_start_epoch) |
| node0_start_string = node0_start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| node0_end_datetime = datetime.datetime.fromtimestamp(node0_down_end_epoch) |
| node0_end_string = node0_end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| |
| # Job names |
| test_job1 = "job1" |
| test_job2 = "job2" |
| test_job3 = "job3" |
| |
| # Job 0 |
| # We want this to look like job1 but run right before hand |
| job0_start_epoch = period_start_epoch |
| job0_duration = 1200 |
| job0_end_epoch = job0_start_epoch + job0_duration |
| |
| # Job 1 |
| job1_start_epoch = job0_end_epoch |
| job1_duration = 2700 |
| job1_end_epoch = job1_start_epoch + job1_duration |
| job1_start_datetime = datetime.datetime.fromtimestamp(job1_start_epoch) |
| job1_start_string = job1_start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| job1_end_datetime = datetime.datetime.fromtimestamp(job1_end_epoch) |
| job1_end_string = job1_end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| job1_duration_datetime = datetime.datetime.fromtimestamp(midnight_epoch + job1_duration) |
| job1_duration_string = job1_duration_datetime.strftime("%H:%M:%S") |
| job1_nodes = node1 |
| job1_cpus = node1_cpus |
| job1_alloc = (job0_duration + job1_duration) * job1_cpus |
| job1_acct = account1 |
| |
| # Job 2 |
| # Make job eligible an hour into the allocation |
| job2_elig_epoch = period_start_epoch + 3600 |
| # start the job 65 minutes later so we can check reserved time |
| job2_start_epoch = job2_elig_epoch + 3900 |
| # Run for a day |
| job2_duration = 86400 |
| job2_end_epoch = job2_start_epoch + job2_duration |
| job2_start_datetime = datetime.datetime.fromtimestamp(job2_start_epoch) |
| job2_start_string = job2_start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| job2_end_datetime = datetime.datetime.fromtimestamp(job2_end_epoch) |
| job2_end_string = job2_end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| job2_duration_datetime = datetime.datetime.fromtimestamp(midnight_epoch + job2_duration) |
| job2_duration_string = job2_duration_datetime.strftime("%-d-%H:%M:%S") |
| job2_nodes = f"{cluster1}_node[0-1]" |
| job2_cpus = node0_cpus + node1_cpus |
| job2_alloc = job2_duration * job2_cpus |
| job2_acct = account3 |
| |
| # Job 3 |
| # Make job eligible an hour before the end of job2 |
| job3_elig_epoch = job2_end_epoch - 3600 |
| # Start the job at the end of job2 |
| job3_start_epoch = job2_end_epoch |
| # Run for 65 minutes |
| job3_duration = 3900 |
| job3_end_epoch = job3_start_epoch + job3_duration |
| job3_start_datetime = datetime.datetime.fromtimestamp(job3_start_epoch) |
| job3_start_string = job3_start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| job3_end_datetime = datetime.datetime.fromtimestamp(job3_end_epoch) |
| job3_end_string = job3_end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| job3_duration_datetime = datetime.datetime.fromtimestamp(midnight_epoch + job3_duration) |
| job3_duration_string = job3_duration_datetime.strftime("%H:%M:%S") |
| # Run on just node0 |
| job3_nodes = node0 |
| job3_cpus = node0_cpus |
| job3_alloc = job3_duration * job3_cpus |
| job3_acct = account2 |
| |
| # Cred Allocations |
| acct1_alloc = job1_alloc |
| acct3_alloc = job2_alloc |
| acct2_alloc = acct3_alloc + job3_alloc |
| total_alloc = job1_alloc + job2_alloc + job3_alloc |
| wckey1_alloc = job1_alloc + job2_alloc + job3_alloc |
| user1_wckey1_alloc = job1_alloc + job3_alloc |
| user2_wckey1_alloc = job2_alloc |
| |
| # Association dictionaries (populated up in create_entities fixture) |
| user_account_id = {} |
| user_wckey_id = {} |
| |
| # Federation |
| federation1 = "federation1" |
| |
| |
| # Setup |
| @pytest.fixture(scope="module", autouse=True) |
| def setup(): |
| atf.require_accounting(modify=True) |
| atf.require_config_parameter("TrackWCKey", "yes", source="slurmdbd") |
| atf.require_slurm_running() |
| |
| |
| @pytest.fixture(scope="module") |
| def create_entities(): |
| """Populate accounting database with entities (clusters, accounts, users, ...)""" |
| |
| # Create accounting entities |
| atf.run_command( |
| f"sacctmgr -i add cluster {cluster1}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.run_command( |
| f"sacctmgr -i add cluster {cluster2}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.run_command( |
| f"sacctmgr -i add account {account1} cluster={cluster1},{cluster2}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.run_command( |
| f"sacctmgr -i add account {account2} cluster={cluster1},{cluster2}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.run_command( |
| f"sacctmgr -i add account {account3} cluster={cluster1},{cluster2} parent={account2}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.run_command( |
| f"sacctmgr -i add user user={user1} cluster={cluster1},{cluster2} account={account1},{account2},{account3} wckey={wckey1}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.run_command( |
| f"sacctmgr -i add user user={user2} cluster={cluster1},{cluster2} account={account1},{account2},{account3} wckey={wckey1}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| |
| # Populate user_account_id dictionary with user-account association ids |
| output = atf.run_command_output( |
| f'sacctmgr -n -P list assoc users={user1},{user2} account={account1},{account2},{account3} cluster={cluster2} format="user,account,id"', |
| fatal=True, |
| ) |
| for line in output.splitlines(): |
| if match := re.search(r"^([^|]+)\|([^|]+)\|(\d+)$", line): |
| user, account, id = match.group(1, 2, 3) |
| if user not in user_account_id: |
| user_account_id[user] = {} |
| user_account_id[user][account] = id |
| # Verify they were all populated with an id |
| for user in user1, user2: |
| if user not in user_account_id: |
| atf.log_die(f"Account association for {user} was not created") |
| for account in account1, account2, account3: |
| if account not in user_account_id[user]: |
| atf.log_die(f"Association for {user} and {account} was not created") |
| |
| # Populate user_wckey_id dictionary with user-wckey association ids |
| output = atf.run_command_output( |
| f'sacctmgr -n -P list wckeys users={user1},{user2} wckeys={wckey1} cluster={cluster1} format="user,wckey,id"', |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| for line in output.splitlines(): |
| if match := re.search(r"^([^|]+)\|([^|]+)\|(\d+)$", line): |
| user, wckey, id = match.group(1, 2, 3) |
| if user not in user_wckey_id: |
| user_wckey_id[user] = {} |
| user_wckey_id[user][wckey] = id |
| # Verify they were all populated with an id |
| for user in user1, user2: |
| if user not in user_wckey_id: |
| atf.log_die(f"WCKey association for {user} was not created") |
| if wckey1 not in user_wckey_id[user]: |
| atf.log_die(f"Association for {user} and {wckey1} was not created") |
| |
| |
| @pytest.fixture(scope="module") |
| def archive_load(create_entities): |
| """Populate accounting database with cluster utilization data""" |
| |
| # Cluster utilization |
| for cluster in cluster1, cluster2: |
| sql_input_file = f"{cluster}.sql" |
| sql_input_path = f"{str(atf.module_tmp_path / sql_input_file)}" |
| with open(sql_input_path, "w") as f: |
| # Add utilization data for a period prior to today's date |
| # We are using 'Mon Dec 31 23:00:00 2007' = 1199167200 as the start |
| f.write( |
| f"insert into cluster_event_table (node_name, cluster, tres, period_start, period_end, reason, cluster_nodes) values ('', '{cluster}', '1={cluster_cpus}', {period_start_epoch}, {period_end_epoch}, 'Cluster processor count', '{node_list}')" |
| ) |
| # Mark a node down for 30 minutes starting at 45 minutes after the start to make sure our rollups work so we should get 15 minutes on one hour and 15 on the other |
| f.write( |
| f", ('{node0}', '{cluster}', '1={node0_cpus}', {node0_down_start_epoch}, {node0_down_end_epoch}, 'down','')" |
| ) |
| f.write( |
| " on duplicate key update period_start=VALUES(period_start), period_end=VALUES(period_end);\n" |
| ) |
| # Now we will put in a job running for an hour and 5 minutes |
| f.write( |
| "insert into job_table (jobid, associd, wckey, wckeyid, uid, gid, `partition`, blockid, cluster, account, eligible, submit, start, end, suspended, name, state, comp_code, priority, req_cpus, tres_alloc, nodelist, kill_requid, qos, deleted) values" |
| ) |
| f.write( |
| f" ('65536', '{user_account_id[user1][account1]}', '{wckey1}', '{user_wckey_id[user1][wckey1]}', '{uid}', '{gid}', 'debug', '', '{cluster}', '{job1_acct}', {job0_start_epoch}, {job0_start_epoch}, {job0_start_epoch}, {job0_end_epoch}, '0', '{test_job1}', '3', '0', '{job1_cpus}', {job1_cpus}, '1={job1_cpus}', '{job1_nodes}', '0', '0', '0')" |
| ) |
| f.write( |
| f", ('65537', '{user_account_id[user1][account1]}', '{wckey1}', '{user_wckey_id[user1][wckey1]}', '{uid}', '{gid}', 'debug', '', '{cluster}', '{job1_acct}', {job1_start_epoch}, {job1_start_epoch}, {job1_start_epoch}, {job1_end_epoch}, '0', '{test_job1}', '3', '0', '{job1_cpus}', {job1_cpus}, '1={job1_cpus}', '{job1_nodes}', '0', '0', '0')" |
| ) |
| f.write( |
| f", ('65538', '{user_account_id[user2][account3]}', '{wckey1}', '{user_wckey_id[user2][wckey1]}', '{uid}', '{gid}', 'debug', '', '{cluster}', '{job2_acct}', {job2_elig_epoch}, {job2_elig_epoch}, {job2_start_epoch}, {job2_end_epoch}, '0', '{test_job2}', '3', '0', '{job2_cpus}', {job2_cpus}, '1={job2_cpus}', '{job2_nodes}', '0', '0', '0')" |
| ) |
| f.write( |
| f", ('65539', '{user_account_id[user1][account2]}', '{wckey1}', '{user_wckey_id[user1][wckey1]}', '{uid}', '{gid}', 'debug', '', '{cluster}', '{job3_acct}', {job3_elig_epoch}, {job3_elig_epoch}, {job3_start_epoch}, {job3_end_epoch}, '0', '{test_job3}', '3', '0', {job3_cpus}, '{job3_cpus}', '1={job3_cpus}', '{job3_nodes}', '0', '0', '0')" |
| ) |
| f.write( |
| " on duplicate key update id=LAST_INSERT_ID(id), eligible=VALUES(eligible), submit=VALUES(submit), start=VALUES(start), end=VALUES(end), associd=VALUES(associd), tres_alloc=VALUES(tres_alloc), wckey=VALUES(wckey), wckeyid=VALUES(wckeyid);\n" |
| ) |
| |
| # Perform archive load |
| atf.run_command( |
| f"sacctmgr -i -n archive load {sql_input_path}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| |
| # Use sacct to see if the job loaded |
| output = atf.run_command_output( |
| f"sacct -P -M {cluster} --format=cluster,account,associd,wckey,wckeyid,start,end,elapsed --noheader --start={period_start_string} --end={period_end_string}", |
| fatal=True, |
| ) |
| # Verify values |
| if not re.search( |
| rf"{cluster}\|{account1}\|{user_account_id[user1][account1]}\|{wckey1}\|{user_wckey_id[user1][wckey1]}\|{job1_start_string}\|{job1_end_string}\|{job1_duration_string}", |
| output, |
| ): |
| atf.log_die("The job accounting data was not loaded correctly for job1") |
| if not re.search( |
| rf"{cluster}\|{account3}\|{user_account_id[user2][account3]}\|{wckey1}\|{user_wckey_id[user2][wckey1]}\|{job2_start_string}\|{job2_end_string}\|{job2_duration_string}", |
| output, |
| ): |
| atf.log_die("The job accounting data was not loaded correctly for job2") |
| if not re.search( |
| rf"{cluster}\|{account2}\|{user_account_id[user1][account2]}\|{wckey1}\|{user_wckey_id[user1][wckey1]}\|{job3_start_string}\|{job3_end_string}\|{job3_duration_string}", |
| output, |
| ): |
| atf.log_die("The job accounting data was not loaded correctly for job3") |
| |
| # Use sacctmgr to see if the node event loaded |
| output = atf.run_command_output( |
| f"sacctmgr -P list events cluster={cluster} format=cluster,noden,start,end,cpu --noheader start={period_start_string} end={period_end_string}", |
| fatal=True, |
| ) |
| # Verify values |
| if not re.search( |
| rf"{cluster}\|\|{period_start_string}\|{period_end_string}\|{cluster_cpus}", |
| output, |
| ): |
| atf.log_die( |
| "The event accounting data was not loaded correctly for the cluster" |
| ) |
| if not re.search( |
| rf"{cluster}\|{node0}\|{node0_start_string}\|{node0_end_string}\|{node0_cpus}", |
| output, |
| ): |
| atf.log_die("The event accounting data was not loaded correctly for node0") |
| |
| # Use sacctmgr to roll up the time period |
| atf.run_command_output( |
| f"sacctmgr -i roll {period_start_string} {period_end_string}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| |
| |
| @pytest.fixture(scope="class") |
| def configure_federation(): |
| """Configuration for federation tests""" |
| |
| # Create federation1 |
| atf.run_command( |
| f"sacctmgr -i add federation {federation1}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.run_command( |
| f"sacctmgr -i mod federation {federation1} set clusters={cluster1},{cluster2}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| |
| # Set the ClusterName to cluster1 |
| original_cluster_name = atf.get_config_parameter("ClusterName") |
| atf.set_config_parameter("ClusterName", cluster1) |
| |
| yield |
| |
| # Undo the above |
| atf.run_command( |
| f"sacctmgr -i delete federation {federation1}", |
| user=atf.properties["slurm-user"], |
| fatal=True, |
| ) |
| atf.set_config_parameter("ClusterName", original_cluster_name) |
| |
| |
| @pytest.mark.usefixtures("create_entities", "archive_load", "configure_federation") |
| class TestFederation: |
| """Test federated sreport functionality on second hour cluster1 usage""" |
| |
| # Set class variables |
| |
| cluster = f"FED:{federation1}" |
| # Second hour start: Tue Jan 1 00:00:00 2008 |
| start_datetime = datetime.datetime(2008, 1, 1, 0, 0, 0) |
| start_epoch = int(start_datetime.timestamp()) |
| start_string = start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| # Second hour end: Tue Jan 1 01:00:00 2008 |
| end_datetime = datetime.datetime(2008, 1, 1, 1, 0, 0) |
| end_epoch = int(end_datetime.timestamp()) |
| end_string = end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| |
| reported_duration = (end_epoch - start_epoch) * cluster_cpus * 2 |
| down_duration = (node0_down_end_epoch - start_epoch) * node0_cpus * 2 |
| allocated_duration = (job1_end_epoch - start_epoch) * job1_cpus * 2 |
| wckey_allocated_duration = allocated_duration |
| reserved_duration = (end_epoch - job2_elig_epoch) * job2_cpus * 2 |
| # Use the same logic inside the plugin to figure out the correct idle and reserved durations |
| idle_duration = reported_duration - ( |
| down_duration + allocated_duration + reserved_duration |
| ) |
| if idle_duration < 0: |
| reserved_duration = reserved_duration + idle_duration |
| idle_duration = 0 |
| if reserved_duration < 0: |
| reserved_duration = 0 |
| |
| down_string = f"{down_duration}({float(down_duration*100)/reported_duration:.2f}%)" |
| allocated_string = ( |
| f"{allocated_duration}({float(allocated_duration*100)/reported_duration:.2f}%)" |
| ) |
| reserved_string = ( |
| f"{reserved_duration}({float(reserved_duration*100)/reported_duration:.2f}%)" |
| ) |
| idle_string = f"{idle_duration}({float(idle_duration*100)/reported_duration:.2f}%)" |
| reported_string = f"{reported_duration}({100:.2f}%)" |
| |
| def test_cluster_utilization(self): |
| """Test utilization report for second hour""" |
| |
| command = f"sreport --federation cluster utilization start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,idle,down,alloc,res,reported" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{self.idle_string}|{self.down_string}|{self.allocated_string}|{self.reserved_string}|{self.reported_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_AccountUtilizationByUser(self): |
| """Test cluster AccountUtilizationByUser report""" |
| |
| command = f"sreport --federation cluster AccountUtilizationByUser start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|root||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_UserUtilizationByWckey(self): |
| """Test cluster UserUtilizationByWckey report""" |
| |
| command = f"sreport --federation cluster UserUtilizationByWckey start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{user1}||{wckey1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_WckeyUtilizationByUser(self): |
| """Test cluster WckeyUtilizationByUser report""" |
| |
| command = f"sreport --federation cluster WckeyUtilizationByUser start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{wckey1}|||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{wckey1}|{user1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_user_top(self): |
| """Test User Top report""" |
| |
| command = f"sreport --federation user top start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{user1}||{account1}|{self.allocated_string}|{self.idle_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_size(self): |
| """Test Job Size report""" |
| |
| command = f"sreport --federation job size AcctAsParent grouping=2,4 start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{account1}|0|{self.allocated_duration}|0|{self.allocated_duration}|100.00%" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_sizesbywckey(self): |
| """Test Job sizesbywckey report""" |
| |
| command = f"sreport --federation job sizesbywckey grouping=2,4 start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{wckey1}|0|{self.wckey_allocated_duration}|0|{self.wckey_allocated_duration}|100.00%" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| |
| @pytest.mark.usefixtures("create_entities", "archive_load") |
| class TestFirstHour: |
| """Test cluster usage for the first hour""" |
| |
| # Set class variables |
| |
| # First hour start: Mon Dec 31 23:00:00 2007 |
| start_datetime = datetime.datetime(2007, 12, 31, 23, 0, 0) |
| start_epoch = int(start_datetime.timestamp()) |
| start_string = start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| # Second hour end: Tue Jan 1 00:00:00 2008 |
| end_datetime = datetime.datetime(2008, 1, 1, 0, 0, 0) |
| end_epoch = int(end_datetime.timestamp()) |
| end_string = end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| |
| reported_duration = (end_epoch - start_epoch) * cluster_cpus |
| down_duration = (end_epoch - node0_down_start_epoch) * node0_cpus |
| allocated_duration = (end_epoch - job0_start_epoch) * node1_cpus |
| wckey_allocated_duration = allocated_duration |
| reserved_duration = 0 |
| idle_duration = reported_duration - ( |
| down_duration + allocated_duration + reserved_duration |
| ) |
| |
| down_string = f"{down_duration}({float(down_duration*100)/reported_duration:.2f}%)" |
| allocated_string = ( |
| f"{allocated_duration}({float(allocated_duration*100)/reported_duration:.2f}%)" |
| ) |
| reserved_string = ( |
| f"{reserved_duration}({float(reserved_duration*100)/reported_duration:.2f}%)" |
| ) |
| idle_string = f"{idle_duration}({float(idle_duration*100)/reported_duration:.2f}%)" |
| reported_string = f"{reported_duration}({100:.2f}%)" |
| |
| def test_cluster_utilization(self): |
| """Test cluster utilization report for first hour""" |
| |
| command = f"sreport -M{cluster2} cluster utilization cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,idle,down,alloc,res,reported" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster1}|{self.idle_string}|{self.down_string}|{self.allocated_string}|{self.reserved_string}|{self.reported_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{self.idle_string}|{self.down_string}|{self.allocated_string}|{self.reserved_string}|{self.reported_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_UserUtilizationByAccount(self): |
| """Test cluster UserUtilizationByAccount report""" |
| |
| command = f"sreport -M{cluster2} cluster UserUtilizationByAccount start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,login,account,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster2}|{user1}|{account1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport -M{cluster2} cluster UserUtilizationByAccount cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,login,account,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster1}|{user1}|{account1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{user1}|{account1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_AccountUtilizationByUser(self): |
| """Test cluster AccountUtilizationByUser report""" |
| |
| command = f"sreport -M{cluster2} cluster AccountUtilizationByUser start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster2}|root||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{account1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport -M{cluster2} cluster AccountUtilizationByUser cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster1}|root||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster1}|{account1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster1}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|root||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{account1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_UserUtilizationByWckey(self): |
| """Test cluster UserUtilizationByWckey report""" |
| |
| command = f"sreport -M{cluster2} cluster UserUtilizationByWckey start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,login,wckey,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{cluster2}|{user1}||{wckey1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport -M{cluster2} cluster UserUtilizationByWckey cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,login,wckey,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{cluster1}|{user1}||{wckey1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{user1}||{wckey1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_WckeyUtilizationByUser(self): |
| """Test cluster WckeyUtilizationByUser report""" |
| |
| command = f"sreport -M{cluster2} cluster WckeyUtilizationByUser start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,wckey,login,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{cluster2}|{wckey1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{wckey1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport -M{cluster2} cluster WckeyUtilizationByUser cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,wckey,login,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{cluster1}|{wckey1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster1}|{wckey1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{wckey1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{wckey1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_user_top(self): |
| """Test User Top report""" |
| |
| command = f"sreport -M{cluster2} user top start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster2}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport -M{cluster2} user top cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster1}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_size(self): |
| """Test Job Size report""" |
| |
| command = f"sreport -M{cluster2} job size AcctAsParent grouping=2,4 start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster2}|{account1}|0|{self.allocated_duration}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport --local job size AcctAsParent grouping=2,4 cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{cluster1}|{account1}|0|{self.allocated_duration}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_sizesbywckey(self): |
| """Test Job sizesbywckey report""" |
| |
| command = f"sreport -M{cluster2} job sizesbywckey grouping=2,4 start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{cluster2}|{wckey1}|0|{self.wckey_allocated_duration}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport -M{cluster2} job sizesbywckey grouping=2,4 cluster='{cluster1}' start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{cluster1}|{wckey1}|0|{self.wckey_allocated_duration}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{cluster2}|{wckey1}|0|{self.wckey_allocated_duration}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| |
| @pytest.mark.usefixtures("create_entities", "archive_load") |
| class TestSecondHour: |
| """Test cluster usage for the second hour""" |
| |
| # Set class variables |
| |
| # Since there are 2 test clusters we will just use one |
| cluster = cluster1 |
| |
| # Second hour start: Tue Jan 1 00:00:00 2008 |
| start_datetime = datetime.datetime(2008, 1, 1, 0, 0, 0) |
| start_epoch = int(start_datetime.timestamp()) |
| start_string = start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| # Second hour end: Tue Jan 1 01:00:00 2008 |
| end_datetime = datetime.datetime(2008, 1, 1, 1, 0, 0) |
| end_epoch = int(end_datetime.timestamp()) |
| end_string = end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| |
| reported_duration = (end_epoch - start_epoch) * cluster_cpus |
| down_duration = (node0_down_end_epoch - start_epoch) * node0_cpus |
| allocated_duration = (job1_end_epoch - start_epoch) * job1_cpus |
| wckey_allocated_duration = allocated_duration |
| reserved_duration = (end_epoch - job2_elig_epoch) * job2_cpus |
| idle_duration = reported_duration - ( |
| down_duration + allocated_duration + reserved_duration |
| ) |
| # Use the same logic inside the plugin to figure out the correct idle and reserved durations |
| if idle_duration < 0: |
| reserved_duration = reserved_duration + idle_duration |
| idle_duration = 0 |
| if reserved_duration < 0: |
| reserved_duration = 0 |
| |
| down_string = f"{down_duration}({float(down_duration*100)/reported_duration:.2f}%)" |
| allocated_string = ( |
| f"{allocated_duration}({float(allocated_duration*100)/reported_duration:.2f}%)" |
| ) |
| reserved_string = ( |
| f"{reserved_duration}({float(reserved_duration*100)/reported_duration:.2f}%)" |
| ) |
| idle_string = f"{idle_duration}({float(idle_duration*100)/reported_duration:.2f}%)" |
| reported_string = f"{reported_duration}({100:.2f}%)" |
| |
| def test_cluster_utilization(self): |
| """Test cluster utilization report for second hour""" |
| |
| command = f"sreport --local cluster utilization cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,idle,down,alloc,res,reported" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{self.idle_string}|{self.down_string}|{self.allocated_string}|{self.reserved_string}|{self.reported_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_AccountUtilizationByUser(self): |
| """Test cluster AccountUtilizationByUser report""" |
| |
| command = f"sreport --local cluster AccountUtilizationByUser cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|root||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_UserUtilizationByWckey(self): |
| """Test cluster UserUtilizationByWckey report""" |
| |
| command = f"sreport --local cluster UserUtilizationByWckey cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,login,wckey,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{user1}|{wckey1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_WckeyUtilizationByUser(self): |
| """Test cluster WckeyUtilizationByUser report""" |
| |
| command = f"sreport --local cluster WckeyUtilizationByUser cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,wckey,login,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{wckey1}||{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{wckey1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_user_top(self): |
| """Test User Top report""" |
| |
| command = f"sreport --local user top cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{account1}|{user1}|{self.allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_size(self): |
| """Test Job Size report""" |
| |
| command = f"sreport --local job size AcctAsParent grouping=2,4 cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{account1}|0|{self.allocated_duration}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_sizesbywckey(self): |
| """Test Job sizesbywckey report""" |
| |
| command = f"sreport --local job sizesbywckey grouping=2,4 cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{wckey1}|0|{self.wckey_allocated_duration}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| |
| @pytest.mark.usefixtures("create_entities", "archive_load") |
| class TestFirstThreeDays: |
| """Test cluster usage for the first 3 days""" |
| |
| # Set class variables |
| |
| # Since there are 2 test clusters we will just use one |
| cluster = cluster1 |
| |
| # First 3 Days Start: Mon Dec 31 23:00:00 2007 |
| start_datetime = datetime.datetime(2007, 12, 31, 23, 0, 0) |
| start_epoch = int(start_datetime.timestamp()) |
| start_string = start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| # First 3 Days End: Tue Jan 3 00:00:00 2008 |
| end_datetime = datetime.datetime(2008, 1, 3, 0, 0, 0) |
| end_epoch = int(end_datetime.timestamp()) |
| end_string = end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| |
| reported_duration = (end_epoch - start_epoch) * cluster_cpus |
| down_duration = (node0_down_end_epoch - node0_down_start_epoch) * node0_cpus |
| allocated_duration = ( |
| ((job1_end_epoch - job0_start_epoch) * job1_cpus) |
| + ((job2_end_epoch - job2_start_epoch) * job2_cpus) |
| + ((job3_end_epoch - job3_start_epoch) * job3_cpus) |
| ) |
| wckey_allocated_duration1 = job1_alloc + job3_alloc |
| wckey_allocated_duration2 = job2_alloc |
| reserved_duration = ((job2_start_epoch - job2_elig_epoch) * job2_cpus) + ( |
| (job3_start_epoch - job3_elig_epoch) * job3_cpus |
| ) |
| # I didn't have time to do the correct math here so I am just putting in 9000 which should be the correct value of over commit |
| overcommit_duration = 9000 |
| reserved_duration -= overcommit_duration |
| idle_duration = reported_duration - ( |
| down_duration + allocated_duration + reserved_duration |
| ) |
| # Use the same logic inside the plugin to figure out the correct idle and reserved durations |
| if idle_duration < 0: |
| reserved_duration = reserved_duration + idle_duration |
| idle_duration = 0 |
| if reserved_duration < 0: |
| reserved_duration = 0 |
| |
| down_string = f"{down_duration}({float(down_duration*100)/reported_duration:.2f}%)" |
| allocated_string = ( |
| f"{allocated_duration}({float(allocated_duration*100)/reported_duration:.2f}%)" |
| ) |
| reserved_string = ( |
| f"{reserved_duration}({float(reserved_duration*100)/reported_duration:.2f}%)" |
| ) |
| idle_string = f"{idle_duration}({float(idle_duration*100)/reported_duration:.2f}%)" |
| overcommit_string = f"{overcommit_duration}({float(overcommit_duration*100)/reported_duration:.2f}%)" |
| reported_string = f"{reported_duration}({100:.2f}%)" |
| |
| job1_allocated_string = ( |
| f"{job1_alloc}({float(job1_alloc*100)/reported_duration:.2f}%)" |
| ) |
| job2_allocated_string = ( |
| f"{job2_alloc}({float(job2_alloc*100)/reported_duration:.2f}%)" |
| ) |
| job3_allocated_string = ( |
| f"{job3_alloc}({float(job3_alloc*100)/reported_duration:.2f}%)" |
| ) |
| total_allocated_string = ( |
| f"{total_alloc}({float(total_alloc*100)/reported_duration:.2f}%)" |
| ) |
| account1_allocated_string = ( |
| f"{acct1_alloc}({float(acct1_alloc*100)/reported_duration:.2f}%)" |
| ) |
| account2_allocated_string = ( |
| f"{acct2_alloc}({float(acct2_alloc*100)/reported_duration:.2f}%)" |
| ) |
| account3_allocated_string = ( |
| f"{acct3_alloc}({float(acct3_alloc*100)/reported_duration:.2f}%)" |
| ) |
| wckey1_allocated_string = ( |
| f"{wckey1_alloc}({float(wckey1_alloc*100)/reported_duration:.2f}%)" |
| ) |
| user1_wckey1_allocated_string = ( |
| f"{user1_wckey1_alloc}({float(user1_wckey1_alloc*100)/reported_duration:.2f}%)" |
| ) |
| user2_wckey1_allocated_string = ( |
| f"{user2_wckey1_alloc}({float(user2_wckey1_alloc*100)/reported_duration:.2f}%)" |
| ) |
| |
| def test_cluster_utilization(self): |
| """Test cluster utilization report for first 3 days""" |
| |
| command = f"sreport --local cluster utilization cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,idle,down,alloc,res,over,reported" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{self.idle_string}|{self.down_string}|{self.allocated_string}|{self.reserved_string}|{self.overcommit_string}|{self.reported_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_UserUtilizationByAccount(self): |
| """Test cluster UserUtilizationByAccount report""" |
| |
| command = f"sreport --local cluster UserUtilizationByAccount cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,login,account,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{user2}|{account3}|{self.job2_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{user1}|{account1}|{self.job1_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{user1}|{account2}|{self.job3_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_AccountUtilizationByUser(self): |
| """Test cluster AccountUtilizationByUser report""" |
| |
| command = f"sreport --local cluster AccountUtilizationByUser cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|root||{self.total_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account1}||{self.account1_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account1}|{user1}|{self.job1_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account2}||{self.account2_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account2}|{user1}|{self.job3_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account3}||{self.account3_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account3}|{user2}|{self.job2_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_UserUtilizationByWckey(self): |
| """Test cluster UserUtilizationByWckey report""" |
| |
| command = f"sreport --local cluster UserUtilizationByWckey cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,login,wckey,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = ( |
| rf"{self.cluster}|{user2}|{wckey1}|{self.user2_wckey1_allocated_string}" |
| ) |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = ( |
| rf"{self.cluster}|{user1}|{wckey1}|{self.user1_wckey1_allocated_string}" |
| ) |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_cluster_WckeyUtilizationByUser(self): |
| """Test cluster WckeyUtilizationByUser report""" |
| |
| command = f"sreport --local cluster WckeyUtilizationByUser cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,wckey,login,used" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{wckey1}||{self.wckey1_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = ( |
| rf"{self.cluster}|{wckey1}|{user1}|{self.user1_wckey1_allocated_string}" |
| ) |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = ( |
| rf"{self.cluster}|{wckey1}|{user2}|{self.user2_wckey1_allocated_string}" |
| ) |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_user_top(self): |
| """Test User Top report""" |
| |
| command = f"sreport --local user top cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n format=cluster,account,login,used" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{account3}|{user2}|{self.job2_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account1}|{user1}|{self.job1_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account2}|{user1}|{self.job3_allocated_string}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_size(self): |
| """Test Job Size report""" |
| |
| command = f"sreport --local job size AcctAsParent grouping=2,4 cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{account1}|0|{job1_alloc}|0" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| pattern = rf"{self.cluster}|{account2}|0|{job3_alloc}|{job2_alloc}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| command = f"sreport --local job size AcctAsParent grouping=2,4 cluster='{self.cluster}' account='{account2}' start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output(command, fatal=True) |
| pattern = rf"{self.cluster}|{account3}|0|0|{job2_alloc}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| def test_job_sizesbywckey(self): |
| """Test Job sizesbywckey report""" |
| |
| command = f"sreport --local job sizesbywckey grouping=2,4 cluster='{self.cluster}' start={self.start_string} end={self.end_string} -tsecper -P -n" |
| output = atf.run_command_output( |
| command, user=atf.properties["slurm-user"], fatal=True |
| ) |
| pattern = rf"{self.cluster}|{wckey1}|0|{self.wckey_allocated_duration1}|{self.wckey_allocated_duration2}" |
| assert ( |
| re.search(pattern, output) is not None |
| ), f'Command output for "{command}" did not match expected pattern "{pattern}"' |
| |
| |
| @pytest.mark.usefixtures("create_entities", "archive_load") |
| class TestSpecificJobs: |
| """Test for jobs that ran on a node at a certain time""" |
| |
| # Set class variables |
| |
| # Since there are 2 test clusters we will just use one |
| cluster = cluster1 |
| |
| def test_job1(self): |
| """Search for job1 on cluster1""" |
| |
| output = atf.run_command_output( |
| f"sacct -p -M {self.cluster} --state=completed --start={job1_start_string} --end={job1_end_string} --format=jobname", |
| fatal=True, |
| ) |
| assert ( |
| len(re.findall(rf"{test_job1}", output)) == 2 |
| ), "Job 1 not found (twice) in sacct" |
| |
| def test_job2(self): |
| """Search for job2 on cluster1""" |
| |
| output = atf.run_command_output( |
| f"sacct -p -M {self.cluster} --state=completed --start={job2_start_string} --end={job2_end_string} --format=jobname", |
| fatal=True, |
| ) |
| assert ( |
| len(re.findall(rf"{test_job2}", output)) == 1 |
| ), "Job 2 not found (once) in sacct" |
| |
| def test_job3(self): |
| """Search for job3 on cluster1""" |
| |
| output = atf.run_command_output( |
| f"sacct -p -M {self.cluster} --state=completed --start={job3_start_string} --end={job3_end_string} --format=jobname", |
| fatal=True, |
| ) |
| assert ( |
| len(re.findall(rf"{test_job3}", output)) == 1 |
| ), "Job 3 not found (once) in sacct" |
| |
| |
| @pytest.mark.usefixtures("create_entities", "archive_load") |
| class TestOptions: |
| """Get job usage reports with default options and with different switches""" |
| |
| # Set class variables |
| |
| # Start: Mon Dec 31 23:00:00 2007 |
| start_datetime = datetime.datetime(2007, 12, 31, 23, 0, 0) |
| start_epoch = int(start_datetime.timestamp()) |
| start_string = start_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| # End: Tue Dec 31 23:59:59 2008 |
| end_datetime = datetime.datetime(2008, 12, 31, 23, 59, 59) |
| end_epoch = int(end_datetime.timestamp()) |
| end_string = end_datetime.strftime("%Y-%m-%dT%H:%M:%S") |
| |
| def test_job_sizesbyaccount_default(self): |
| """Test job report for default (non-AcctAsParent)""" |
| |
| output = atf.run_command_output( |
| f"sreport job sizesbyaccount printjobcount cluster='{cluster1}' start={self.start_string} end={self.end_string} -P -n", |
| fatal=True, |
| ) |
| assert re.search(rf"{cluster1}|root|4|0|0|0|0|100.00%", output) is not None |
| |
| def test_job_sizesbyaccount_flatview(self): |
| """Test job report with flatview""" |
| |
| output = atf.run_command_output( |
| f"sreport job sizesbyaccount printjobcount cluster='{cluster1}' start={self.start_string} end={self.end_string} -P -n flatview", |
| fatal=True, |
| ) |
| # test22-1clus1|test22-1acct1|2|0|0|0|0|50.00% |
| # test22-1clus1|test22-1acct2|1|0|0|0|0|25.00% |
| # test22-1clus1|test22-1acct3|1|0|0|0|0|25.00% |
| assert ( |
| re.search( |
| rf"{cluster1}|{account1}|2|0|0|0|0|50.00%\n{cluster1}|{account2}|1|0|0|0|0|25.00\n{cluster1}|{account3}|1|0|0|0|0|25.00", |
| output, |
| ) |
| is not None |
| ) |
| |
| def test_specific_account(self): |
| """Test the job report for specific account""" |
| |
| # Should show just one account with all of its jobs |
| output = atf.run_command_output( |
| f"sreport job sizesbyaccount printjobcount cluster='{cluster1}' account={account1} start={self.start_string} end={self.end_string} -P -n", |
| fatal=True, |
| ) |
| assert ( |
| re.search(rf"{cluster1}|{account1}|2|0|0|0|0|100.00%", output) is not None |
| ) |
