Health test configuration file reference for Cluster Validation Suite (CVS)

2026-10-26

9 min read time

Applies to Linux

The health configuration file controls AGFHC, TransferBench, and ROCm Validation Suite (RVS) burn-in parameters. Use these tests to validate GPU hardware, memory bandwidth, and compute throughput on each cluster node before production workloads.

See Run AGFHC, TransferBench, and RVS health burn-in tests in Cluster Validation Suite (CVS) for more information on running these tests.

Note

In this configuration file, {user-id} resolves to the current username at runtime. You can also manually change this value to your username.

health_config.json

{

    "agfhc":
    {
        "path": "/opt/amd/agfhc",
        "package_tar_ball": "/home/{user-id}/PACKAGES/agfhc-mi300x_1.22.0_ub2204.tar.bz2",
        "install_dir": "/home/{user-id}/INSTALL/agfhc/",
        "_comment_env": "Companion-tool paths. Defaults match an /opt/amd install. Update every value when path changes. A missing key falls back to the directory next to path. FIREXS2_APP_PATH and LD_LIBRARY_PATH are derived from FIREXS_PATH.",
        "env": {
            "TXB_PATH": "/opt/amd/transferbench",
            "MINIHPL_PATH": "/opt/amd/minihpl",
            "MEMROCKER_PATH": "/opt/amd/memrocker",
            "FIREXS_PATH": "/opt/amd/firexs2",
            "AFIDSAG_PATH": "/opt/amd/afid"
        },
        "_comments_log_dir": "log_dir has to be a NON NFS local file system. CVS runs mkdir -p and test -w on each node before the recipes.",
        "log_dir": "/root/agfhc_logs",
        "hbm_test_duration": "00:01:30"
    },
    "transferbench":
    {
      "path": "/home/{user-id}/INSTALL/TransferBench",
      "git_install_path": "/home/{user-id}/INSTALL/",
      "git_url": "https://github.com/ROCm/TransferBench.git",
      "git_tag": "v1.67.00",
      "_comment_rocm_path": "ROCm installation path. Set the placeholder changeme to auto-detect from /opt/rocm or /opt/rocm/core-*",
      "rocm_path": "<changeme>",
      "_comment_a2asweep_timeout": "Orch.exec timeout seconds for the a2asweep preset. Default 1500 covers the shipped 24-combo sweep plus buffer. Override directly, or omit and derive via a2asweep_combos / a2asweep_sec_per_combo / a2asweep_timeout_buffer.",
      "a2asweep_timeout": 1500,
      "results":
      {
          "bytes_to_transfer": "268435456",
          "path": "/home/{user-id}/INSTALL/TransferBench",
          "_comment_gpu_to_gpu_a2a_per_link_min": "Required a2a guide bar (GB/s): every GPU RTotal_i/(N-1) >= 32.9. Legacy alias gpu_to_gpu_a2a_avg is accepted.",
          "gpu_to_gpu_a2a_per_link_min": "32.9",
          "_comment_gpu_to_gpu_a2a_rtotal": "If set, Run Deck plots raw RTotal with no threshold. Not used for pass/fail.",
          "gpu_to_gpu_a2a_rtotal": "320.0",
          "avg_gpu_to_gpu_p2p_unidir_bw": "33.9",
          "avg_gpu_to_gpu_p2p_bidir_bw": "43.9",
          "best_gpu0_bw": "480.0",
          "32_cu_local_read": "1650",
          "32_cu_local_write": "1250.0",
          "32_cu_local_copy": "1250.0",
          "32_cu_rem_read": "48.0",
          "32_cu_rem_write": "48.0",
          "32_cu_rem_copy": "48.0"

      }
    },
    "rvs":
    {
        "path": "<changeme>/bin",
        "git_install_path": "/home/{user-id}/INSTALL/rvs",
        "git_url": "https://github.com/ROCm/ROCmValidationSuite.git",
        "nfs_install": "True",
        "_comment_rocm_path": "ROCm installation path.Set to placeholder changeme auto-detect from /opt/rocm or /opt/rocm/core-*",
        "rocm_path": "<changeme>",
        "config_path_mi300x": "/opt/rocm/share/rocm-validation-suite/conf/MI300X",
        "config_path_default": "/opt/rocm/share/rocm-validation-suite/conf",
        "_comment_rvs_test_level": "RVS test level configuration (0-5). 0: Run individual tests (skip level test), 1-5: Run LEVEL config test if RVS >= 1.3.0, else run individual tests. Default is 4.",
        "rvs_test_level": 4,
        "tests": [
            {
                "name": "level_config",
                "description": "RVS LEVEL Configuration Test - Runs all modules collectively",
                "timeout": 14400,
                "expected_pass": true,
                "fail_regex_patterns": [
                    "met:\\s*FALSE",
                    "pass:\\s*FALSE",
                    "\\[ERROR\\s*\\]",
                    "FAIL",
                    "ERROR:",
                    "peqt false",
                    "RVS-ERROR",
                    "Missing packages\\s*:\\s*([1-9]\\d*)",
                    "Version mismatch packages\\s*:\\s*([1-9]\\d*)"
                ]
            },
            {
                "name": "mem_test",
                "config_file": "mem.conf",
                "description": "Memory Test",
                "timeout": 10000,
                "expected_pass": true,
                "fail_regex_pattern": "FAIL|\\[ERROR\\s*\\]|RVS-ERROR"
            },
            {
                "name": "gst_single",
                "config_file": "gst_single.conf",
                "description": "GPU Stress Test",
                "timeout": 18000,
                "expected_pass": true,
                "fail_regex_pattern": "met:\\s*FALSE|RVS-ERROR"
            },
            {
                "name": "iet_stress",
                "config_file": "iet_stress.conf",
                "description": "Peak Power Test",
                "timeout": 3600,
                "expected_pass": true,
                "fail_regex_pattern": "pass:\\s*FALSE|RVS-ERROR"
            },
            {
                "name": "pebb_single",
                "config_file": "pebb_single.conf",
                "description": "PCI Express Bandwidth Benchmark",
                "timeout": 3600,
                "expected_pass": true,
                "fail_regex_pattern": "\\[ERROR\\s*\\]|RVS-ERROR"
            },
            {
                "name": "pbqt_single",
                "config_file": "pbqt_single.conf",
                "description": "P2P Benchmark and Qualification Tool",
                "timeout": 3600,
                "expected_pass": true,
                "fail_regex_pattern": "FAIL|ERROR:|RVS-ERROR"
            },
            {
                "name": "babel_stream",
                "config_file": "babel.conf",
                "description": "BABEL Benchmark Test",
                "timeout": 9000,
                "expected_pass": true,
                "fail_regex_pattern": "\\[ERROR\\s*\\]|RVS-ERROR"
            }
        ]
    }

}

Parameters

The following parameters are available in the health configuration file.

AGFHC

AGFHC configuration parameters and their defaults are listed below.

Configuration parameters	Default values	Description
path	/opt/amd/agfhc	Directory that contains the agfhc executable. Any other path runs ./install --rocm-tar -o on the parent of path, without sudo. That parent must be writable on the nodes.
package_tar_ball	/home/{user-id}/ PACKAGES/agfhc-mi300x _1.22.0_ub2204.tar.bz2	Path where the tar ball is downloaded
install_dir	/home/{user-id}/INSTALL/agfhc/	Path where AGFHC runs
env	TXB_PATH=/opt/amd/transferbench, MINIHPL_PATH=/opt/amd/minihpl, MEMROCKER_PATH=/opt/amd/memrocker, FIREXS_PATH=/opt/amd/firexs2, AFIDSAG_PATH=/opt/amd/afid	Companion-tool paths exported for every AGFHC run, including /opt/amd. Update them when path changes. A missing key falls back to <parent of path>/transferbench, minihpl, memrocker, firexs2, or <path>/sag for AFIDSAG_PATH. FIREXS2_APP_PATH and LD_LIBRARY_PATH are derived from FIREXS_PATH.
log_dir	/root/agfhc_logs	Local, non-NFS log directory. CVS runs mkdir -p and test -w on each node before the recipes.
hbm_test_duration	00:01:30	HBM test duration

TransferBench

TransferBench configuration parameters and their defaults are listed below.

Configuration parameters	Default values	Description
path	/opt/amd/transferbench	Path where TransferBench is installed
git_install_path	/home/{user-id}/INSTALL/	Path where the Git repo is installed
git_url	ROCm/TransferBench.git	URL for Git repo
git_tag	v1.67.00	Git tag to checkout after cloning TransferBench
rocm_path	<changeme>	Set the path of rocm
a2asweep_timeout	1500	Orchestrator timeout in seconds for test_transfer_bench_a2asweep. When set, this value wins. When omitted, CVS derives the timeout as a2asweep_combos × a2asweep_sec_per_combo + a2asweep_timeout_buffer (defaults 24, 50, 300 → 1500), and never returns less than 600 (the former hard-coded 10-minute limit). Raise this on slower nodes or larger custom sweep ranges so a healthy sweep is not aborted mid-run.
a2asweep_combos	24	Optional. Combo count used only when a2asweep_timeout is omitted. Matches the default TransferBench a2asweep table size (block size × unroll combinations).
a2asweep_sec_per_combo	50	Optional. Seconds assumed per combo when deriving a2asweep_timeout.
a2asweep_timeout_buffer	300	Optional. Extra seconds added after combos × sec_per_combo when deriving a2asweep_timeout.
bytes_to_transfer	268435456	Amount of data to transfer in bytes (256 MB); this is the payload size for bandwidth tests
gpu_to_gpu_a2a_per_link_min	32.9	Required guide bar for test_transfer_bench_a2a (GB/s). For each GPU, RTotal_i / (N-1) must be at least this value (coverage-matrix bar 32.9). Pytest and Run Deck both require this key (or the legacy alias gpu_to_gpu_a2a_avg). Benign VMware provider No such file lines are ignored by scanning; other provider failures are not.
gpu_to_gpu_a2a_rtotal	320.0	If present, Run Deck also plots raw per-GPU RTotal series without a threshold (annotation only), so chart status cannot disagree with the per-link cell gate. Not used for pass/fail; omitting gpu_to_gpu_a2a_per_link_min fails a2a in both pytest and Run Deck.
avg_gpu_to_gpu_p2p_unidir_bw	33.9	Average peer-to-peer unidirectional bandwidth (GB/s) between GPU pairs
avg_gpu_to_gpu_p2p_bidir_bw	43.9	Average peer-to-peer bidirectional bandwidth (GB/s) between GPU pair
best_gpu0_bw	480.0	Best measured bandwidth (GB/s) for GPU 0
32_cu_local_read	1650	Local memory read bandwidth (GB/s) using 32 CUs
32_cu_local_write	1250.0	Local memory write bandwidth (GB/s) using 32 CUs
32_cu_local_copy	1250.0	Local memory copy bandwidth (GB/s) using 32 CUs
32_cu_rem_read	48.0	Remote memory read bandwidth (GB/s) using 32 CUs
32_cu_rem_write	48.0	Remote memory write bandwidth (GB/s) using 32 CUs
32_cu_rem_copy	48.0	Remote memory copy bandwidth (GB/s) using 32 CUs

ROCm Validation Suite (RVS)

RVS configuration parameters and their defaults are listed below.

Configuration parameters	Default values	Description
path	<changeme>/bin	Path where the RVS test is installed
git_install_path	/home/{user-id}/INSTALL/rvs	Path to installed Git repo
git_url	ROCm/ROCmValidationSuite.git	URL for Git repo
nfs_install	True	Set the flag to install nfs
rocm_path	<changeme>	Set the path of rocm
config_path_mi300x	/opt/rocm/share/ rocm-validation-suite /conf/MI300X	Path for Instinct MI300X configuration
config_path_default	/opt/rocm/share/ rocm-validation -suite/conf	Default path for RVS
rvs_test_level	4	Test level
name	level_config	Test name
description	RVS LEVEL Configuration Test - Runs all modules collectively	Test description
timeout	14400	Timeout in secs
expected_pass	True	Result
fail_regex_pattern	Values: "met:\\s*FALSE", "pass:\\s*FALSE", "\\[ERROR\\s*\\]", "FAIL", "ERROR:", "peqt false", "RVS-ERROR", "Missing packages\\s*:\\s*([1-9]\\d*)", "Version mismatch packages\\s*:\\s*([1-9]\\d*)"	Regular expressions
name	mem_test	Test name
config_file	mem_test.conf	Test config file
description	Memory test	Test description
timeout	10000	Timeout in secs
expected_pass	True	Result
fail_regex_pattern	FAIL|\\[ERROR\\s*\\]|RVS-ERROR	Failure expression
name	gst_single	Test name
config_file	gst_single.conf	Test config file
description	GPU Stress Test	Test description
timeout	18000	Timeout in secs
expected_pass	True	Result
fail_regex_pattern	met:\\s*FALSE|RVS-ERROR	Failure expression
name	iet_stress	Test name
config_file	iet_stress.conf	Test config file
description	Peak Power Test	Test description
timeout	3600	Timeout in secs
expected_pass	True	Result
fail_regex_pattern	\\[ERROR\\s*\\]|RVS-ERROR	Failure expression
name	pbqt_single	Test name
config_file	pbqt_single.conf	Test config file
description	P2P Benchmark and Qualification Tool	Test description
timeout	3600	Timeout in secs
expected_pass	True	Result
fail_regex_pattern	FAIL|ERROR:|RVS-ERROR	Failure expression
name	babel_stream	Test name
config_file	babel_stream.conf	Test config file
description	BABEL Benchmark Test	Test description
timeout	9000	Timeout in secs
expected_pass	True	Result
fail_regex_pattern	\\[ERROR\\s*\\]|RVS-ERROR	Failure expression