From 9499162a844e20a2df25a2673d1e6edbb681de9b Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Thu, 10 Sep 2026 19:52:50 -0700 Subject: [PATCH 1/3] Add GLM-5.2 B200 Dynamo SGLang AgentX configs --- .../b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml | 118 +++++++++++ .../b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml | 118 +++++++++++ .../b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml | 118 +++++++++++ .../disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml | 193 ++++++++++++++++++ .../disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml | 185 +++++++++++++++++ configs/nvidia-master.yaml | 105 ++++++++++ perf-changelog.yaml | 12 ++ runners/launch_b200-nscale-slurm.sh | 21 +- 8 files changed, 868 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml new file mode 100644 index 0000000000..eeb62a7b53 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml @@ -0,0 +1,118 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; agentic_srt.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +name: agg-b200-tp8-c1-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +dynamo: + install: false +backend: + type: sglang + sglang_config: + aggregated: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +health_check: + max_attempts: 1440 + interval_seconds: 10 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml new file mode 100644 index 0000000000..cda6220659 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml @@ -0,0 +1,118 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; agentic_srt.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +name: agg-b200-tp8-c4-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +dynamo: + install: false +backend: + type: sglang + sglang_config: + aggregated: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +health_check: + max_attempts: 1440 + interval_seconds: 10 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml new file mode 100644 index 0000000000..f94a5e6e10 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml @@ -0,0 +1,118 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; agentic_srt.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +name: agg-b200-tp8-c8-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +dynamo: + install: false +backend: + type: sglang + sglang_config: + aggregated: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +health_check: + max_attempts: 1440 + interval_seconds: 10 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml new file mode 100644 index 0000000000..19c888fcb1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml @@ -0,0 +1,193 @@ +name: disagg-b200-1p1d-dep8-dep8-c64-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_prefill: 8 + prefill_nodes: 1 + prefill_workers: 1 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: false +backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_DSV4_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_DSV4_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + decode: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 + chunked-prefill-size: 64 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutedsl + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.85 + disaggregation-decode-extra-slots: 0 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml new file mode 100644 index 0000000000..5b3eb86cf7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml @@ -0,0 +1,185 @@ +name: disagg-b200-1p4d-dep8-tp4-c48-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 + decode_nodes: 2 + decode_workers: 4 + gpus_per_decode: 4 + gpus_per_prefill: 8 + prefill_nodes: 1 + prefill_workers: 1 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: false +backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_DSV4_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_DSV4_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + decode: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + chunked-prefill-size: 64 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 526fbec480..a8e97b870e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9137,6 +9137,111 @@ glm5.2-fp4-b200-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } +# GLM-5.2 NVFP4 B200 AgentX on Dynamo + SGLang. EAGLE uses the model's +# built-in nextn head, with acceptance pinned to the golden thinking-on AL in +# golden_al_distribution/glm5.2_mtp.yaml. Both arms use HiCache write-back +# DRAM offload for the agentic working set. +glm5.2-fp4-b200-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b200-nscale + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.5.0.dev20260909" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml" + - spec-decoding: mtp + conc-list: [4] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml" + - spec-decoding: mtp + conc-list: [8] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml" + +glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b200-nscale + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.5.0.dev20260909" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # 1P(DEP8)4D(TP4): three nodes total. + - spec-decoding: mtp + conc-list: [48] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1P(DEP8)1D(DEP8): two nodes total. + - spec-decoding: mtp + conc-list: [64] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:v0.5.17-cu130 model: nvidia/GLM-5.2-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b0dc6a14d4..fce14e64ea 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7145,3 +7145,15 @@ - "Update SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, build commit sgl-project/sglang@30705c00) to the v0.5.19 release image lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, build commit sgl-project/sglang@0bcd822377da7b5718e674eaf9c870d349424dd1, Docker Hub last pushed 2026-09-04T22:50:19Z)." - "The release image ships the same CUDA 13.0.3, FlashInfer 0.6.18 and sgl-kernel 0.4.6.post1 as the nightly. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. TP8/EP1 DRAM HiCache concurrency 2 through 24 unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2966 + +- config-keys: + - glm5.2-fp4-b200-dynamo-sglang-agentic-agg + - glm5.2-fp4-b200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add a GLM-5.2 NVFP4 B200 Dynamo-SGLang AgentX curve on the NScale runner." + - "Cover Pareto aggregate TP8 points at concurrency 1, 4, and 8, disaggregated 1P(DEP8)4D(TP4) at concurrency 48, and disaggregated 1P(DEP8)1D(DEP8) at concurrency 64." + - "Use lmsysorg/sglang:nightly-dev-20260910-00840301 with its bundled Dynamo 1.5.0.dev20260909, EAGLE speculative decoding, and 110 GB HiCache write-back DRAM offload per prefill rank." + - "Allow up to 1800 seconds for NIXL UCX engine initialization while registering the prefill HiCache pools." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5e0ef627c..839c329515 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -5,8 +5,8 @@ # Self-contained because Nscale has its own Slurm and storage layout. # # Scope: multi-node Dynamo-vLLM DeepSeek-V4-Pro and Kimi K2.6 FP4 runs, plus -# DeepSeek-V4-Pro FP4 Dynamo-SGLang STP and MTP runs, on the -# b200-nscale runner label. +# DeepSeek-V4-Pro and GLM-5.2 FP4 Dynamo-SGLang runs, on the b200-nscale +# runner label. # Anything else exits non-zero. SLURM_PARTITION="batch_1" @@ -49,6 +49,10 @@ elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/Kimi-K3}" export SRT_SLURM_MODEL_PREFIX="kimik3" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/GLM-5.2-NVFP4}" + # This alias must match model.path in the checked-in GLM-5.2 recipes. + export SRT_SLURM_MODEL_PREFIX="glm-5.2-fp4" elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" && $FRAMEWORK == "tilert" ]]; then export SRT_SLURM_MODEL_PREFIX="glm5.1-fp8" else @@ -58,6 +62,7 @@ fi if [[ $FRAMEWORK != "dynamo-vllm" ]] && [[ $MODEL_PREFIX != "dsv4" || $PRECISION != "fp4" || $FRAMEWORK != "dynamo-sglang" || ( $SPEC_DECODING != "none" && $SPEC_DECODING != "mtp" ) ]] && + [[ $MODEL_PREFIX != "glm5.2" || $PRECISION != "fp4" || $FRAMEWORK != "dynamo-sglang" || $SPEC_DECODING != "mtp" ]] && [[ $MODEL_PREFIX != "glm5.1" || $PRECISION != "fp8" || $FRAMEWORK != "tilert" || $SPEC_DECODING != "mtp" ]]; then run_compat_launcher fi @@ -130,6 +135,18 @@ elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ recipes/vllm/kimi-k3/agentic || exit 1 +elif [[ $MODEL_PREFIX == "glm5.2" && $FRAMEWORK == "dynamo-sglang" ]]; then + # Pin the renderer used for the validated checked-in recipes. + git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || { + echo "Error: srt-slurm v1.0.53 did not resolve to the pinned commit" >&2 + exit 1 + } + mkdir -p recipes/sglang/glm5.2/b200-fp4/agentic || exit 1 + cp -rT \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic" \ + recipes/sglang/glm5.2/b200-fp4/agentic || exit 1 elif [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 From 97ccdd4cab5b2aab3b8e58f249c64c8e36df4450 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Thu, 10 Sep 2026 19:56:02 -0700 Subject: [PATCH 2/3] Link performance changelog to PR 2995 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fce14e64ea..ac280f9465 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7156,4 +7156,4 @@ - "Cover Pareto aggregate TP8 points at concurrency 1, 4, and 8, disaggregated 1P(DEP8)4D(TP4) at concurrency 48, and disaggregated 1P(DEP8)1D(DEP8) at concurrency 64." - "Use lmsysorg/sglang:nightly-dev-20260910-00840301 with its bundled Dynamo 1.5.0.dev20260909, EAGLE speculative decoding, and 110 GB HiCache write-back DRAM offload per prefill rank." - "Allow up to 1800 seconds for NIXL UCX engine initialization while registering the prefill HiCache pools." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2995 From c4c3458cc37ff5265c61fa458b8379963ef5d424 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Fri, 11 Sep 2026 01:44:40 -0700 Subject: [PATCH 3/3] Extend c48 NIXL initialization timeout --- .../b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml index 73b14d7648..084b6b791e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml @@ -44,7 +44,7 @@ backend: DYN_SKIP_SGLANG_LOG_FORMATTING: '1' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' @@ -65,7 +65,7 @@ backend: DYN_SKIP_SGLANG_LOG_FORMATTING: '1' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7c5f216489..e31a60b903 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7189,5 +7189,5 @@ - "Add a GLM-5.2 NVFP4 B200 Dynamo-SGLang AgentX curve on the NScale runner." - "Cover Pareto aggregate TP8 points at concurrency 1, 4, and 8, disaggregated 1P(DEP8)4D(TP4) at concurrency 48, and disaggregated 1P(DEP8)1D(DEP8) at concurrency 64." - "Use lmsysorg/sglang:nightly-dev-20260910-00840301 with its bundled Dynamo 1.5.0.dev20260909, EAGLE speculative decoding, and 110 GB HiCache write-back DRAM offload per prefill rank." - - "Allow up to 1800 seconds for NIXL UCX engine initialization while registering the prefill HiCache pools." + - "Allow up to 3600 seconds for NIXL UCX engine initialization in the 1P(DEP8)4D(TP4) configuration while registering its larger set of HiCache pools." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2995