From 58730432a89a2a9b57ccca29b652922b3b715ac9 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Wed, 24 Jun 2026 11:16:54 -0700 Subject: [PATCH 1/3] Bump dsv4-fp4-b200-sglang image to SGLang nightly 20260624 --- .github/configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 77833a1b3..75347ac19 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1719,7 +1719,7 @@ dsr1-fp4-b200-sglang-mtp: - { tp: 8, ep: 1, conc-start: 4, conc-end: 512, spec-decoding: mtp } dsv4-fp4-b200-sglang: - image: lmsysorg/sglang:deepseek-v4-blackwell@sha256:df18bfc4aa9ecf59451002b49ba00cae58042de9e2a96378bbd21b404dd62c7b + image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b200-dsv4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d6a5f35e4..97e5013c1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4153,3 +4153,9 @@ - "Run the PR #1891 MiniMax-M3 MXFP8 B300 Dynamo-vLLM recipe set on top of current main." - "Uses the vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 image and the TEP4/TEP8 8k1k topologies not covered by PR #1890." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1891 + +- config-keys: + - dsv4-fp4-b200-sglang + description: + - "Bump SGLang image from lmsysorg/sglang:deepseek-v4-blackwell (digest sha256:df18bfc4...) to mainline nightly lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2." + pr-link: XXX From 12a209f9955bd8124ccbc95d21cef44927c95c59 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Wed, 24 Jun 2026 11:17:21 -0700 Subject: [PATCH 2/3] Update perf-changelog pr-link for #1923 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 97e5013c1..c9ee134f7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4158,4 +4158,4 @@ - dsv4-fp4-b200-sglang description: - "Bump SGLang image from lmsysorg/sglang:deepseek-v4-blackwell (digest sha256:df18bfc4...) to mainline nightly lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2." - pr-link: XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1923 From 01dcb28c7ce340bffe21a38e4fb68041d6a65d08 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 29 Jun 2026 10:36:29 -0700 Subject: [PATCH 3/3] Apply B200 Agg config from shyeh25@0988d8a7 Picks up the dsv4-fp4-b200-sglang DP-attention max-throughput conc-end extension (512 -> 1024) and the megamoe-based dsv4_fp4_b200.sh refactor. Co-Authored-By: Claude Opus 4.7 --- .github/configs/nvidia-master.yaml | 4 +- .../fixed_seq_len/dsv4_fp4_b200.sh | 64 +++++++++++++------ perf-changelog.yaml | 2 +- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 75347ac19..b99fdb8f5 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1719,7 +1719,7 @@ dsr1-fp4-b200-sglang-mtp: - { tp: 8, ep: 1, conc-start: 4, conc-end: 512, spec-decoding: mtp } dsv4-fp4-b200-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + image: lmsysorg/sglang:nightly-dev-cu13-20260628-da802ddc model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b200-dsv4 @@ -1754,7 +1754,7 @@ dsv4-fp4-b200-sglang: # DP-attention (DP_ATTENTION=true) — balanced CONC range - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 128 } # DP-attention (DP_ATTENTION=true) — max-throughput CONC range - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512 } + - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } dsv4-fp4-b200-vllm: image: vllm/vllm-openai:nightly-3f0a91bb96f8d72e0498b95c166e817deae14d62 diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200.sh index e1d031854..898402776 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200.sh @@ -21,12 +21,7 @@ if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi nvidia-smi # Common SGLANG env vars (apply to every config). -export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 -export SGLANG_OPT_USE_JIT_NORM=1 -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 -export SGLANG_OPT_USE_TOPK_V2=1 -export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 # TODO(Cam): the lmsysorg/sglang:deepseek-v4-blackwell image installs sglang # editable at /workspace/sglang/python; prior sglang tags used /sgl-workspace/sglang. @@ -35,6 +30,7 @@ export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 # Drop the runner conditional once lmsys moves sglang back out of /workspace. SERVER_LOG="$PWD/server.log" +PORT=${PORT:-8888} echo "TP: $TP, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" @@ -46,31 +42,61 @@ fi start_gpu_monitor --output "$PWD/gpu_metrics.csv" -# Pick the parallelism + MoE backend based on DP_ATTENTION (mirrors the vllm -# script's pattern). DP-attention turns on EP-MoE (deepep) and the related -# mega_moe optimizations; single-instance uses flashinfer_mxfp4. -DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' +# 1k inputs need more SWA cache headroom than 8k inputs do. +if [[ "$ISL" == "1024" ]]; then + SWA_FULL_TOKENS_RATIO=0.5 +else + SWA_FULL_TOKENS_RATIO=0.1 +fi +# Pick the parallelism + MoE backend based on DP_ATTENTION. DP-attention turns on +# EP-MoE (megamoe) + the mega_moe / mixed-chunk optimizations; single-instance +# uses flashinfer_mxfp4. if [ "${DP_ATTENTION}" = "true" ]; then - export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 - export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 + export SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION=8 + export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 export SGLANG_OPT_USE_FAST_MASK_EP=1 export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 + export NVSHMEM_DISABLE_IB=1 + export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 + export SGLANG_OPT_USE_ONLINE_COMPRESS=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=2048 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 + export SGLANG_EXPERIMENTAL_ENABLE_PIECEWISE_CUDA_GRAPH_MOE_A2A=1 + export NCCL_MNNVL_ENABLE=1 + export NCCL_CUMEM_ENABLE=1 + export MC_FORCE_MNNVL=1 + export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True + + MEM_FRACTION_STATIC=0.835 + MAX_RUNNING_REQUESTS=4352 + SWA_FULL_TOKENS_RATIO=0.12 + PARALLEL_ARGS=( --dp-size "$TP" --enable-dp-attention - --moe-a2a-backend deepep - --deepep-config "$DEEPEP_CONFIG" - --chunked-prefill-size 32768 + --moe-a2a-backend megamoe + --cuda-graph-max-bs 544 + --enable-mixed-chunk + --chunked-prefill-size 16384 + --max-prefill-tokens 16384 + --tokenizer-worker-num 8 + --stream-interval 30 + --enable-prefill-delayer ) else + MEM_FRACTION_STATIC=0.90 + MAX_RUNNING_REQUESTS=512 PARALLEL_ARGS=( --moe-runner-backend flashinfer_mxfp4 --chunked-prefill-size 8192 --disable-flashinfer-autotune + --cuda-graph-max-bs 512 + --tokenizer-worker-num 8 + --stream-interval 30 + --enable-prefill-delayer ) fi @@ -90,9 +116,9 @@ PYTHONNOUSERSITE=1 sglang serve \ --trust-remote-code \ --tp $TP \ --disable-radix-cache \ - --max-running-requests "$((CONC * 3 / 2))" \ - --mem-fraction-static 0.90 \ - --swa-full-tokens-ratio 0.1 \ + --max-running-requests "$MAX_RUNNING_REQUESTS" \ + --mem-fraction-static "$MEM_FRACTION_STATIC" \ + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" \ "${PARALLEL_ARGS[@]}" $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c9ee134f7..f8ce0dc72 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4157,5 +4157,5 @@ - config-keys: - dsv4-fp4-b200-sglang description: - - "Bump SGLang image from lmsysorg/sglang:deepseek-v4-blackwell (digest sha256:df18bfc4...) to mainline nightly lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2." + - "Bump SGLang image from lmsysorg/sglang:deepseek-v4-blackwell (digest sha256:df18bfc4...) to mainline nightly lmsysorg/sglang:nightly-dev-cu13-20260628-da802ddc." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1923