From dd89caede26d2212e55bc03b692d59e2d7da23ef Mon Sep 17 00:00:00 2001
From: Christian Sonnabend <sonnabendch@gmail.com>
Date: Mon, 8 Jun 2026 14:59:02 +0200
Subject: [PATCH 1/6] Avoiding numactl execution to avoid crashes of FST in
 container env without accesibility to numactl

---
 prodtests/full-system-test/start_tmux.sh | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/prodtests/full-system-test/start_tmux.sh b/prodtests/full-system-test/start_tmux.sh
index 18f019573904f..e5d07e4d5f825 100755
--- a/prodtests/full-system-test/start_tmux.sh
+++ b/prodtests/full-system-test/start_tmux.sh
@@ -147,10 +147,17 @@ else
   : ${CALIB_TASKS:=""}
 fi
 
+if command -v numactl >/dev/null 2>&1; then
+  NUMACTL_CMD="numactl --interleave=all"
+else
+  NUMACTL_CMD=""
+fi
+
 if [ "0$FST_TMUX_BATCH_MODE" == "01" ]; then
   { sleep $FST_SLEEP0; eval "NUMAID=0 $GEN_TOPO_MYDIR/dpl-workflow.sh ${LOGCMD/\[REPLACE]/0}"; eval "$ENDCMD"; } &
   { sleep $FST_SLEEP1; eval "NUMAID=1 $GEN_TOPO_MYDIR/dpl-workflow.sh ${LOGCMD/\[REPLACE]/1}"; eval "$ENDCMD"; } &
-  { sleep $FST_SLEEP2; eval "SEVERITY=debug numactl --interleave=all $GEN_TOPO_MYDIR/$CMD ${LOGCMD/\[REPLACE]/2}"; eval "$KILLCMD $ENDCMD"; } &
+  { sleep $FST_SLEEP2; eval "SEVERITY=debug $NUMACTL_CMD $GEN_TOPO_MYDIR/$CMD ${LOGCMD/\[REPLACE]/2}"; eval "$KILLCMD $ENDCMD"; } &
+
   for i in $CALIB_TASKS; do
     { eval "AGGREGATOR_TASKS=$i $CALIB_COMMAND ${LOGCMD/\[REPLACE]/3_${i}}"; eval "$ENDCMD"; } &
   done
@@ -162,7 +169,9 @@ else
   for i in `seq 1 $(($NUM_DPL_WORKFLOWS - 1))`; do
     TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"sleep $FST_SLEEP1; NUMAID=$i $GEN_TOPO_MYDIR/dpl-workflow.sh ${LOGCMD/\[REPLACE]/1}; $ENDCMD\" ';'"
   done
-  TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"sleep $FST_SLEEP2; SEVERITY=debug numactl --interleave=all $GEN_TOPO_MYDIR/$CMD; $KILLCMD $ENDCMD\" ';'"
+
+  TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"sleep $FST_SLEEP2; SEVERITY=debug $NUMACTL_CMD $GEN_TOPO_MYDIR/$CMD; $KILLCMD $ENDCMD\" ';'"
+
   FIRST_CALIB=1
   for i in $CALIB_TASKS; do
     TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"AGGREGATOR_TASKS=$i $CALIB_COMMAND ${LOGCMD/\[REPLACE]/3_${i}}; $ENDCMD\" ';'"

From 95f3190a4c76b8f65f4d3ad8fdebf62df1c2d0f9 Mon Sep 17 00:00:00 2001
From: Christian Sonnabend <sonnabendch@gmail.com>
Date: Fri, 12 Jun 2026 09:17:02 +0200
Subject: [PATCH 2/6] Adding GPU benchmark scripts and python analysis script

---
 .../analyze_gpu_benchmarks.py                 | 324 ++++++++++++++++
 .../gen_single_gpu_rtc_benchmark.sh           | 347 ++++++++++++++++++
 2 files changed, 671 insertions(+)
 create mode 100644 prodtests/full-system-test/analyze_gpu_benchmarks.py
 create mode 100755 prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh

diff --git a/prodtests/full-system-test/analyze_gpu_benchmarks.py b/prodtests/full-system-test/analyze_gpu_benchmarks.py
new file mode 100644
index 0000000000000..2320b16994dee
--- /dev/null
+++ b/prodtests/full-system-test/analyze_gpu_benchmarks.py
@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+
+import argparse
+import re
+from pathlib import Path
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+try:
+    from scipy.optimize import curve_fit
+    SCIPY_AVAILABLE = True
+except ImportError:
+    SCIPY_AVAILABLE = False
+
+
+LINE_RE = re.compile(
+    r"\[[^\]]*gpu-reconstruction[^\]]*\]:\s*"
+    r"\[(?P<time>\d{2}:\d{2}:\d{2}(?:\.\d+)?)\]\[INFO\]\s*"
+    r"(?P<kind>Processing timeslice:|Done processing timeslice:)"
+    r"(?P<timeslice>\d+)"
+)
+
+
+def parse_hms_to_seconds(hms: str) -> float:
+    hhmmss, *frac = hms.split(".")
+    h, m, s = map(int, hhmmss.split(":"))
+
+    seconds = h * 3600 + m * 60 + s
+
+    if frac:
+        seconds += float("0." + frac[0])
+
+    return seconds
+
+
+def gaussian(x, amplitude, mean, sigma):
+    return amplitude * np.exp(-0.5 * ((x - mean) / sigma) ** 2)
+
+
+def read_timeslice_durations(logfile: Path):
+    starts = {}
+    ends = {}
+    durations = {}
+
+    day_offset = 0.0
+    previous_raw_timestamp = None
+
+    with logfile.open("r", errors="replace") as f:
+        for line in f:
+            match = LINE_RE.search(line)
+            if not match:
+                continue
+
+            raw_timestamp = parse_hms_to_seconds(match.group("time"))
+
+            # Handle midnight wraparound in log order
+            if (
+                previous_raw_timestamp is not None
+                and raw_timestamp < previous_raw_timestamp
+            ):
+                day_offset += 24 * 3600
+
+            previous_raw_timestamp = raw_timestamp
+            timestamp = raw_timestamp + day_offset
+
+            timeslice = int(match.group("timeslice"))
+            kind = match.group("kind")
+
+            if kind == "Processing timeslice:":
+                starts[timeslice] = timestamp
+
+            elif kind == "Done processing timeslice:":
+                if timeslice not in starts:
+                    continue
+
+                start = starts[timeslice]
+                end = timestamp
+
+                durations[timeslice] = end - start
+                ends[timeslice] = end
+
+    return durations, starts, ends
+
+def warn_about_processing_downtime(starts, ends, tolerance_s=0.001):
+    RED = "\033[91m"
+    YELLOW = "\033[93m"
+    BOLD = "\033[1m"
+    RESET = "\033[0m"
+
+    common_timeslices = sorted(set(starts) & set(ends))
+
+    if len(common_timeslices) <= 4:
+        return set()
+
+    ignored_timeslices = set(common_timeslices[:2] + common_timeslices[-2:])
+    excluded_timeslices = set()
+
+    for ts in common_timeslices:
+        next_ts = ts + 1
+
+        if next_ts not in starts:
+            continue
+
+        # Do not warn if the boundary touches one of the first two or last two
+        # complete timeslices.
+        if ts in ignored_timeslices or next_ts in ignored_timeslices:
+            continue
+
+        gap = starts[next_ts] - ends[ts]
+
+        if gap > tolerance_s:
+            affected = {ts - 1, ts, next_ts, next_ts + 1}
+            affected = {
+                x for x in affected
+                if x in common_timeslices and x not in ignored_timeslices
+            }
+
+            print(
+                f"{YELLOW}{BOLD}WARNING:{RESET} "
+                f"{YELLOW}Processing downtime detected between "
+                f"timeslice {ts} and {next_ts}: "
+                f"end[{ts}] -> start[{next_ts}] gap = {gap * 1000:.3f} ms. "
+                f"Excluding timeslices {sorted(affected)} from calculation.{RESET}",
+                flush=True,
+            )
+
+            excluded_timeslices.update(affected)
+
+        elif gap < -tolerance_s:
+            affected = {ts - 1, ts, next_ts, next_ts + 1}
+            affected = {
+                x for x in affected
+                if x in common_timeslices and x not in ignored_timeslices
+            }
+
+            print(
+                f"{RED}{BOLD}WARNING:{RESET} "
+                f"{RED}Processing overlap or timestamp ordering issue between "
+                f"timeslice {ts} and {next_ts}: "
+                f"start[{next_ts}] is {-gap * 1000:.3f} ms before end[{ts}]. "
+                f"Excluding timeslices {sorted(affected)} from calculation.{RESET}",
+                flush=True,
+            )
+
+            excluded_timeslices.update(affected)
+
+    return excluded_timeslices
+
+def fit_gaussian_to_histogram(values, bins):
+    counts, edges = np.histogram(values, bins=bins)
+    centers = 0.5 * (edges[:-1] + edges[1:])
+
+    sample_mean = np.mean(values)
+    sample_sigma = np.std(values, ddof=1)
+
+    if not SCIPY_AVAILABLE:
+        return None, counts, edges
+
+    nonzero = counts > 0
+    if np.count_nonzero(nonzero) < 3:
+        return None, counts, edges
+
+    x = centers[nonzero]
+    y = counts[nonzero]
+
+    p0 = [np.max(y), sample_mean, sample_sigma]
+
+    try:
+        popt, _ = curve_fit(
+            gaussian,
+            x,
+            y,
+            p0=p0,
+            maxfev=10000,
+            bounds=(
+                [0.0, -np.inf, 1e-12],
+                [np.inf, np.inf, np.inf],
+            ),
+        )
+        return popt, counts, edges
+    except Exception:
+        return None, counts, edges
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Analyze gpu-reconstruction timeslice processing durations."
+    )
+    parser.add_argument("-l", "--logfile", type=Path, help="Path to the log file")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        default=Path("gpu_reconstruction_times.png"),
+        help="Output plot filename",
+    )
+    parser.add_argument(
+        "--bins",
+        type=int,
+        default=50,
+        help="Number of histogram bins",
+    )
+    parser.add_argument(
+        "--unit",
+        choices=["s", "ms"],
+        default="s",
+        help="Plot durations in seconds or milliseconds",
+    )
+    parser.add_argument(
+        "--gap-tolerance-ms",
+        type=float,
+        default=1.0,
+        help="Allowed gap between end of timeslice n and start of timeslice n+1 in ms",
+    )
+
+    args = parser.parse_args()
+
+    durations_by_timeslice, starts_by_timeslice, ends_by_timeslice = read_timeslice_durations(
+        args.logfile
+    )
+
+    excluded_timeslices = warn_about_processing_downtime(
+        starts_by_timeslice,
+        ends_by_timeslice,
+        tolerance_s=args.gap_tolerance_ms / 1000.0,
+    )
+    if excluded_timeslices:
+        print(
+            f"Excluded timeslices due to downtime/overlap: "
+            f"{sorted(excluded_timeslices)}"
+        )
+
+    if len(durations_by_timeslice) < 5:
+        raise RuntimeError(
+            f"Found only {len(durations_by_timeslice)} complete timeslices. "
+            "Need at least 5 to drop first two and last two."
+        )
+
+    timeslices = sorted(durations_by_timeslice)
+    trimmed_timeslices = [
+        ts for ts in timeslices[2:-2]
+        if ts not in excluded_timeslices
+    ]
+
+    values = np.array(
+        [durations_by_timeslice[ts] for ts in trimmed_timeslices],
+        dtype=float,
+    )
+
+    if args.unit == "ms":
+        values *= 1000.0
+        unit_label = "ms"
+    else:
+        unit_label = "s"
+
+    n_total = len(timeslices)
+    n_used = len(values)
+
+    sample_mean = np.mean(values)
+    sample_sigma = np.std(values, ddof=1)
+
+    avg_from_sum = np.mean(values)
+
+    fit_result, counts, edges = fit_gaussian_to_histogram(values, args.bins)
+
+    plt.figure(figsize=(9, 6))
+
+    plt.hist(
+        values,
+        bins=args.bins,
+        histtype="stepfilled",
+        alpha=0.45,
+        label="Timeslice duration distribution",
+    )
+
+    if fit_result is not None:
+        amp, fit_mean, fit_sigma = fit_result
+        xfit = np.linspace(edges[0], edges[-1], 1000)
+        yfit = gaussian(xfit, amp, fit_mean, fit_sigma)
+
+        plt.plot(
+            xfit,
+            yfit,
+            linewidth=2,
+            label=f"Gaussian fit: mean={fit_mean:.4g} {unit_label}, sigma={fit_sigma:.4g} {unit_label}",
+        )
+    else:
+        plt.plot([], [], label="Gaussian fit: unavailable")
+
+    plt.plot(
+        [],
+        [],
+        label=f"Sample: mean={sample_mean:.4g} {unit_label}, sigma={sample_sigma:.4g} {unit_label}",
+    )
+
+    plt.xlabel(f"Processing time per timeslice [{unit_label}]")
+    plt.ylabel("Entries")
+    plt.title("gpu-reconstruction timeslice processing duration")
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(args.output, dpi=150)
+
+    print(f"Input file: {args.logfile}")
+    print(f"Complete timeslices found: {n_total}")
+    print(f"Timeslices used after dropping first/last two: {n_used}")
+    print(f"First used timeslice: {trimmed_timeslices[0]}")
+    print(f"Last used timeslice: {trimmed_timeslices[-1]}")
+    print(f"Average duration = sum(durations)/(processed timeslices - 4): {avg_from_sum:.6g} {unit_label}")
+    print(f"Sample mean: {sample_mean:.6g} {unit_label}")
+    print(f"Sample sigma: {sample_sigma:.6g} {unit_label}")
+
+    if fit_result is not None:
+        _, fit_mean, fit_sigma = fit_result
+        print(f"Gaussian fit mean: {fit_mean:.6g} {unit_label}")
+        print(f"Gaussian fit sigma: {fit_sigma:.6g} {unit_label}")
+    else:
+        print("Gaussian fit failed or scipy is unavailable.")
+
+    print(f"Saved plot to: {args.output}")
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh b/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
new file mode 100755
index 0000000000000..e48d779f9e294
--- /dev/null
+++ b/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
@@ -0,0 +1,347 @@
+#!/usr/bin/env bash
+# Generate or run a single-GPU reconstruction benchmark workflow using dpl-workflow.sh.
+#
+# Main benchmark mode:
+#   BENCHMARK_RUN=1 FILEWORKDIR=/path/to/raw_tf_dir ./gen_single_gpu_rtc_benchmark.sh
+
+set -euo pipefail
+
+rm -rf /dev/shm/fmq*
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Locate original workflow script. Keep the original untouched.
+
+: "${GEN_TOPO_MYDIR:=$(dirname "$(realpath "$0")")}"
+: "${O2_DPL_WORKFLOW:=$GEN_TOPO_MYDIR/dpl-workflow.sh}"
+: "${GEN_TOPO_SOURCE_SCRIPT:=$O2_DPL_WORKFLOW}"   # backward compatibility only
+O2_DPL_WORKFLOW="$GEN_TOPO_SOURCE_SCRIPT"
+
+if [[ ! -f "$O2_DPL_WORKFLOW" ]]; then
+  echo "FATAL: dpl workflow script does not exist: $O2_DPL_WORKFLOW" >&2
+  echo "Set O2_DPL_WORKFLOW=/path/to/dpl-workflow.sh" >&2
+  exit 1
+fi
+
+# Helper lookup must remain compatible with the original script directory.
+export GEN_TOPO_MYDIR="$(dirname "$(realpath "$O2_DPL_WORKFLOW")")"
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Match the normal FST startup environment.
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Benchmark defaults. All can be overridden by exporting variables before calling this script.
+
+export DPL_REPORT_PROCESSING="${DPL_REPORT_PROCESSING:-1}"
+
+export FST_TMUX_NO_EPN="${FST_TMUX_NO_EPN:-1}"
+export WORKFLOW_PARAMETERS="${WORKFLOW_PARAMETERS:-GPU,CTF}"
+export GPUTYPE="${GPUTYPE:-CUDA}"
+export NGPUS=1
+export NUMAGPUIDS=1
+export NUMAID="${NUMAID:-0}"
+
+export O2_GPU_DOUBLE_PIPELINE="${O2_GPU_DOUBLE_PIPELINE:-1}"
+export O2_GPU_RTC="${O2_GPU_RTC:-1}"
+
+export EPNSYNCMODE="${EPNSYNCMODE:-0}"
+export SYNCMODE="${SYNCMODE:-1}"
+export SYNCRAWMODE="${SYNCRAWMODE:-0}"
+
+export TIMEFRAME_RATE_LIMIT="${TIMEFRAME_RATE_LIMIT:-5}"
+export GEN_TOPO_NO_TF_RATE_UPSCALING="${GEN_TOPO_NO_TF_RATE_UPSCALING:-1}"
+
+export DISABLE_ROOT_OUTPUT="${DISABLE_ROOT_OUTPUT:-1}"
+
+# Double pipeline requires zsraw input. Therefore default to raw TF input, not CTF.
+export CTFINPUT="${CTFINPUT:-0}"
+export RAWTFINPUT="${RAWTFINPUT:-1}"
+export DIGITINPUT="${DIGITINPUT:-0}"
+export EXTINPUT="${EXTINPUT:-0}"
+
+export NTIMEFRAMES="${NTIMEFRAMES:--1}"
+export TFLOOP="${TFLOOP:-100}"
+export TFDELAY="${TFDELAY:-0}"
+
+export RUN_BENCHMARK="${RUN_BENCHMARK:-0}"
+
+if [[ -f "$PWD/local_env.sh" ]]; then
+  source "$PWD/local_env.sh"
+fi
+
+export ALICE_O2_FST="${ALICE_O2_FST:-1}"
+
+if [[ -f "$GEN_TOPO_MYDIR/setenv.sh" ]]; then
+  source "$GEN_TOPO_MYDIR/setenv.sh" || {
+    echo "FATAL: setenv.sh failed: $GEN_TOPO_MYDIR/setenv.sh" >&2
+    exit 1
+  }
+else
+  echo "WARNING: setenv.sh not found: $GEN_TOPO_MYDIR/setenv.sh" >&2
+fi
+
+echo "# Alien/JAliEn environment check:"
+echo "#   JALIEN_TOKEN_CERT=${JALIEN_TOKEN_CERT:-}"
+echo "#   JALIEN_TOKEN_KEY=${JALIEN_TOKEN_KEY:-}"
+echo "#   X509_USER_PROXY=${X509_USER_PROXY:-}"
+if command -v alien-token-info >/dev/null 2>&1; then
+  alien-token-info || true
+else
+  echo "#   alien-token-info not found in PATH"
+fi
+echo
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Recover JAliEn token environment if alien-token-info works but token env vars are missing.
+
+if command -v alien-token-info >/dev/null 2>&1; then
+  if alien-token-info >/dev/null 2>&1; then
+    uid="$(id -u)"
+
+    for cert in \
+      "/tmp/jalien_token_${uid}.pem" \
+      "/tmp/jalien_token_${USER}.pem" \
+      "/tmp/tokencert_${uid}.pem" \
+      "/tmp/tokencert_${USER}.pem"
+    do
+      if [[ -f "$cert" ]]; then
+        export JALIEN_TOKEN_CERT="${JALIEN_TOKEN_CERT:-$cert}"
+        break
+      fi
+    done
+
+    for key in \
+      "/tmp/jalien_token_${uid}.key" \
+      "/tmp/jalien_token_${USER}.key" \
+      "/tmp/tokenkey_${uid}.pem" \
+      "/tmp/tokenkey_${USER}.pem"
+    do
+      if [[ -f "$key" ]]; then
+        export JALIEN_TOKEN_KEY="${JALIEN_TOKEN_KEY:-$key}"
+        break
+      fi
+    done
+
+    # Some older tools only look for X509_USER_PROXY.
+    if [[ -z "${X509_USER_PROXY:-}" ]]; then
+      for proxy in \
+        "/tmp/x509up_u${uid}" \
+        "/tmp/x509up_${uid}" \
+        "${JALIEN_TOKEN_CERT:-}"
+      do
+        if [[ -n "$proxy" && -f "$proxy" ]]; then
+          export X509_USER_PROXY="$proxy"
+          break
+        fi
+      done
+    fi
+  fi
+fi
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Benchmark naming / output directory.
+
+: "${BENCH_TAG:=${BENCH_TAG:-$(hostname -s)}}"
+BENCH_STAMP="$(date +%Y%m%d_%H%M%S)"
+: "${OUTDIR:=${BENCHMARK_OUTDIR:-$PWD/single_gpu_rtc_bench_${BENCH_TAG}_${BENCH_STAMP}}}"
+mkdir -p "$OUTDIR"
+RUNDIR="$OUTDIR/run"
+mkdir -p "$RUNDIR"
+
+cleanup_rundir() {
+  if [[ -n "${RUNDIR:-}" && -d "$RUNDIR" ]]; then
+    echo "# Cleaning run dir: $RUNDIR"
+    rm -rf -- "$RUNDIR"
+  fi
+}
+
+trap cleanup_rundir EXIT
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Keep accidental files out of the source/original directory.
+
+# Let O2/core dumps land in the benchmark run directory, not in the original working directory.
+export CORE_DUMP_DIR="${CORE_DUMP_DIR:-$RUNDIR}"
+export O2_CORE_DUMP_DIR="${O2_CORE_DUMP_DIR:-$RUNDIR}"
+export FAIRMQ_SHM_MONITOR_CONFIG="${FAIRMQ_SHM_MONITOR_CONFIG:-}"
+
+# Avoid copying input files unless the caller explicitly requests a copy command.
+if [[ "${BENCH_DISABLE_INPUT_COPY:-1}" == "1" ]]; then
+  unset INPUT_FILE_COPY_CMD || true
+fi
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Library path fixes for common EPN/dev-node issues.
+
+: "${BENCH_USE_SYSTEM_FONT_LIBS:=1}"
+: "${BENCH_AUTO_ROCM_LIBS:=0}"
+
+prepend_ld_path() {
+  local dir="$1"
+  [[ -d "$dir" ]] || return 0
+  case ":${LD_LIBRARY_PATH:-}:" in
+    *":$dir:"*) ;;
+    *) export LD_LIBRARY_PATH="$dir:${LD_LIBRARY_PATH:-}" ;;
+  esac
+}
+
+if [[ "0$BENCH_USE_SYSTEM_FONT_LIBS" == "01" ]]; then
+  prepend_ld_path /usr/lib64
+  prepend_ld_path /lib64
+fi
+
+# ROCm library injection is only useful for HIP runs. Keep it off by default for CUDA/NVIDIA containers,
+# because mixed AMD/NVIDIA hosts can otherwise leak ROCm libraries into LD_LIBRARY_PATH.
+if [[ "${GPUTYPE:-}" == "HIP" && "0$BENCH_AUTO_ROCM_LIBS" == "01" ]]; then
+  if [[ -n "${ROCM_PATH:-}" ]]; then
+    prepend_ld_path "$ROCM_PATH/lib64"
+    prepend_ld_path "$ROCM_PATH/lib"
+  fi
+  for d in /opt/rocm/lib /opt/rocm/lib64 /usr/lib64/rocm /usr/lib/rocm/lib; do
+    prepend_ld_path "$d"
+  done
+fi
+
+if [[ -n "${BENCH_EXTRA_LD_LIBRARY_PATH:-}" ]]; then
+  export LD_LIBRARY_PATH="$BENCH_EXTRA_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"
+fi
+
+# Fail early for the unsupported combination instead of letting o2-gpu-reco-workflow crash later.
+if [[ "${O2_GPU_DOUBLE_PIPELINE:-0}" == "1" ]]; then
+  if [[ "${CTFINPUT:-0}" == "1" ]]; then
+    echo "FATAL: O2_GPU_DOUBLE_PIPELINE=1 is incompatible with CTFINPUT=1 in dpl-workflow.sh." >&2
+    echo "Double pipeline requires o2-gpu-reco-workflow --input-type=zsraw." >&2
+    echo "Use RAWTFINPUT=1 or rawAll.cfg input, or set O2_GPU_DOUBLE_PIPELINE=0 for CTF benchmarking." >&2
+    exit 1
+  fi
+  if [[ "${DIGITINPUT:-0}" == "1" ]]; then
+    echo "FATAL: O2_GPU_DOUBLE_PIPELINE=1 is not suitable for DIGITINPUT=1 in dpl-workflow.sh." >&2
+    echo "Digit input uses zsonthefly and is restricted to NTIMEFRAMES=1." >&2
+    exit 1
+  fi
+fi
+
+# Input checks with clearer messages.
+if [[ "${RAWTFINPUT:-0}" == "1" ]]; then
+  if [[ -z "${FILEWORKDIR:-}" && -z "${INPUT_FILE_LIST:-}" ]]; then
+    echo "FATAL: RAWTFINPUT=1 but neither FILEWORKDIR nor INPUT_FILE_LIST is set." >&2
+    echo "Set FILEWORKDIR=/path/to/raw_tf_dir or INPUT_FILE_LIST=/path/to/o2_*.tf" >&2
+    exit 1
+  fi
+  if [[ -z "${INPUT_FILE_LIST:-}" ]] && ! ls "${FILEWORKDIR}"/o2_*.tf >/dev/null 2>&1; then
+    echo "FATAL: RAWTFINPUT=1 but no raw TF file was found." >&2
+    echo "Looked for: ${FILEWORKDIR}/o2_*.tf" >&2
+    echo "Set FILEWORKDIR=/path/to/raw_tf_dir or INPUT_FILE_LIST=/path/to/o2_*.tf" >&2
+    exit 1
+  fi
+fi
+
+# Check CUDA runtime/device visibility before starting the full workflow.
+if [[ "$GPUTYPE" == "CUDA" ]]; then
+  if ! command -v nvidia-smi >/dev/null 2>&1; then
+    echo "WARNING: GPUTYPE=CUDA but nvidia-smi is not in PATH." >&2
+    echo "If this is an Apptainer/Singularity container, run it with --nv." >&2
+  else
+    nvidia-smi -L >/dev/null 2>&1 || {
+      echo "FATAL: GPUTYPE=CUDA but nvidia-smi cannot see an NVIDIA GPU." >&2
+      echo "If this is an Apptainer/Singularity container, run it with --nv." >&2
+      exit 1
+    }
+  fi
+
+  if ! ldconfig -p 2>/dev/null | grep -q 'libcuda.so.1' && \
+     ! find ${LD_LIBRARY_PATH//:/ } -maxdepth 1 -name 'libcuda.so.1*' 2>/dev/null | grep -q .; then
+    echo "WARNING: GPUTYPE=CUDA but libcuda.so.1 is not visible via ldconfig or LD_LIBRARY_PATH." >&2
+    echo "This usually means the container was not started with --nv, or the host NVIDIA driver is not mounted." >&2
+  fi
+fi
+
+# Check HIP runtime visibility before starting the full workflow.
+if [[ "$GPUTYPE" == "HIP" ]]; then
+  if ! ldconfig -p 2>/dev/null | grep -q 'libamdhip64.so.6' && \
+     ! find ${LD_LIBRARY_PATH//:/ } -maxdepth 1 -name 'libamdhip64.so.6*' 2>/dev/null | grep -q .; then
+    echo "FATAL: GPUTYPE=HIP but libamdhip64.so.6 is not visible." >&2
+    echo "Current LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}" >&2
+    echo "Set ROCM_PATH=/opt/rocm or BENCH_EXTRA_LD_LIBRARY_PATH=/path/to/rocm/lib" >&2
+    exit 1
+  fi
+fi
+
+# A single-GPU benchmark must not enter EPN sync mode, because the workflow intentionally sets NGPUS=4 there.
+if [[ "${EPNSYNCMODE:-0}" == "1" ]]; then
+  echo "FATAL: EPNSYNCMODE=1 is incompatible with the single-GPU RTC benchmark." >&2
+  echo "EPNSYNCMODE=1 makes dpl-workflow.sh set GPUTYPE=HIP and NGPUS=4 by design." >&2
+  echo "Use EPNSYNCMODE=0 for this benchmark." >&2
+  exit 1
+fi
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Print configuration.
+
+echo "# single-GPU RTC benchmark"
+echo "# source script: $O2_DPL_WORKFLOW"
+echo "# output dir:    $OUTDIR"
+echo "# run dir:       $RUNDIR"
+echo "# NGPUS=$NGPUS NUMAGPUIDS=$NUMAGPUIDS NUMAID=$NUMAID GPUTYPE=$GPUTYPE"
+echo "# O2_GPU_DOUBLE_PIPELINE=$O2_GPU_DOUBLE_PIPELINE O2_GPU_RTC=$O2_GPU_RTC"
+echo "# CTFINPUT=$CTFINPUT RAWTFINPUT=$RAWTFINPUT DIGITINPUT=$DIGITINPUT EXTINPUT=$EXTINPUT NTIMEFRAMES=$NTIMEFRAMES TFLOOP=$TFLOOP"
+echo "# FILEWORKDIR=${FILEWORKDIR:-} INPUT_FILE_LIST=${INPUT_FILE_LIST:-}"
+echo "# LD_LIBRARY_PATH font-lib workaround: BENCH_USE_SYSTEM_FONT_LIBS=$BENCH_USE_SYSTEM_FONT_LIBS"
+echo "# ROCm library auto-detect: BENCH_AUTO_ROCM_LIBS=$BENCH_AUTO_ROCM_LIBS (active only when GPUTYPE=HIP)"
+echo
+
+export WORKFLOWMODE="print"
+cmdfile="$OUTDIR/workflow_${BENCH_TAG}_${BENCH_STAMP}.sh"
+echo "# Generating workflow only; command file: $cmdfile"
+(
+  cd "$RUNDIR"
+  "$O2_DPL_WORKFLOW"
+) > "$cmdfile"
+
+if [[ "$RUN_BENCHMARK" == "1" ]]; then
+  export WORKFLOWMODE="${WORKFLOWMODE:-run}"
+  log="$OUTDIR/reco_${BENCH_TAG}_${BENCH_STAMP}.log"
+  env | sort > "$OUTDIR/env_${BENCH_TAG}_${BENCH_STAMP}.txt"
+  echo "# Running benchmark; log: $log"
+
+  set +e
+  (
+    cd "$RUNDIR"
+    chmod +x "$cmdfile"
+    /usr/bin/time -v "$cmdfile"
+  ) > "$log" 2>&1
+  status=$?
+  set -e
+
+  # {
+  #   echo "tag,host,date,ngpus,numagpuids,double_pipeline,rtc,ntf,tfloop,elapsed_seconds,max_rss_kb,exit_status"
+  #   awk '
+  #     /Elapsed \(wall clock\) time/ {elapsed=$NF}
+  #     /Maximum resident set size/ {rss=$NF}
+  #     /Exit status/ {status=$NF}
+  #     END {print ENVIRON["BENCH_TAG"] "," ENVIRON["HOSTNAME"] "," strftime("%FT%T%z") "," ENVIRON["NGPUS"] "," ENVIRON["NUMAGPUIDS"] "," ENVIRON["O2_GPU_DOUBLE_PIPELINE"] "," ENVIRON["O2_GPU_RTC"] "," ENVIRON["NTIMEFRAMES"] "," ENVIRON["TFLOOP"] "," elapsed "," rss "," status}' "$log"
+  # } > "$OUTDIR/summary_${BENCH_TAG}_${BENCH_STAMP}.csv"
+
+  # echo "# Summary: $OUTDIR/summary_${BENCH_TAG}_${BENCH_STAMP}.csv"
+  echo "# Full log: $log"
+
+  # --------------------------------------------------------------------------------------------------------------------
+  # Analyze gpu-reconstruction processing timeslice timing and write PNG next to the log.
+
+  : "${GPU_RECO_ANALYZER:=$O2_ROOT/prodtests/full-system-test/analyze_gpu_benchmarks.py}"
+
+  if [[ -f "$GPU_RECO_ANALYZER" ]]; then
+    analysis_png="${log%.log}_gpu_reconstruction_times.png"
+
+    echo "# Analyzing gpu-reconstruction timeslices"
+    echo "# analyzer: $GPU_RECO_ANALYZER"
+    echo "# plot:     $analysis_png"
+
+    python3 "$GPU_RECO_ANALYZER" --logfile "$log" --output "$analysis_png" || {
+      echo "WARNING: gpu-reconstruction timing analysis failed" >&2
+    }
+  else
+    echo "WARNING: gpu-reconstruction analyzer not found: $GPU_RECO_ANALYZER" >&2
+  fi
+
+  exit "$status"
+fi

From 478d76bb68a4fbc787bbb67c4eb8ee3da7c921f1 Mon Sep 17 00:00:00 2001
From: Christian Sonnabend <sonnabendch@gmail.com>
Date: Fri, 12 Jun 2026 09:20:54 +0200
Subject: [PATCH 3/6] Resetting start_tumx.sh to upstream/dev

---
 prodtests/full-system-test/start_tmux.sh | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/prodtests/full-system-test/start_tmux.sh b/prodtests/full-system-test/start_tmux.sh
index e5d07e4d5f825..18f019573904f 100755
--- a/prodtests/full-system-test/start_tmux.sh
+++ b/prodtests/full-system-test/start_tmux.sh
@@ -147,17 +147,10 @@ else
   : ${CALIB_TASKS:=""}
 fi
 
-if command -v numactl >/dev/null 2>&1; then
-  NUMACTL_CMD="numactl --interleave=all"
-else
-  NUMACTL_CMD=""
-fi
-
 if [ "0$FST_TMUX_BATCH_MODE" == "01" ]; then
   { sleep $FST_SLEEP0; eval "NUMAID=0 $GEN_TOPO_MYDIR/dpl-workflow.sh ${LOGCMD/\[REPLACE]/0}"; eval "$ENDCMD"; } &
   { sleep $FST_SLEEP1; eval "NUMAID=1 $GEN_TOPO_MYDIR/dpl-workflow.sh ${LOGCMD/\[REPLACE]/1}"; eval "$ENDCMD"; } &
-  { sleep $FST_SLEEP2; eval "SEVERITY=debug $NUMACTL_CMD $GEN_TOPO_MYDIR/$CMD ${LOGCMD/\[REPLACE]/2}"; eval "$KILLCMD $ENDCMD"; } &
-
+  { sleep $FST_SLEEP2; eval "SEVERITY=debug numactl --interleave=all $GEN_TOPO_MYDIR/$CMD ${LOGCMD/\[REPLACE]/2}"; eval "$KILLCMD $ENDCMD"; } &
   for i in $CALIB_TASKS; do
     { eval "AGGREGATOR_TASKS=$i $CALIB_COMMAND ${LOGCMD/\[REPLACE]/3_${i}}"; eval "$ENDCMD"; } &
   done
@@ -169,9 +162,7 @@ else
   for i in `seq 1 $(($NUM_DPL_WORKFLOWS - 1))`; do
     TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"sleep $FST_SLEEP1; NUMAID=$i $GEN_TOPO_MYDIR/dpl-workflow.sh ${LOGCMD/\[REPLACE]/1}; $ENDCMD\" ';'"
   done
-
-  TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"sleep $FST_SLEEP2; SEVERITY=debug $NUMACTL_CMD $GEN_TOPO_MYDIR/$CMD; $KILLCMD $ENDCMD\" ';'"
-
+  TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"sleep $FST_SLEEP2; SEVERITY=debug numactl --interleave=all $GEN_TOPO_MYDIR/$CMD; $KILLCMD $ENDCMD\" ';'"
   FIRST_CALIB=1
   for i in $CALIB_TASKS; do
     TMUX_COMMAND+=" $TMUX_SPLIT_COMMAND \"AGGREGATOR_TASKS=$i $CALIB_COMMAND ${LOGCMD/\[REPLACE]/3_${i}}; $ENDCMD\" ';'"

From f87246f0eca418b9afa88766465b0ef9568a4fc2 Mon Sep 17 00:00:00 2001
From: Christian Sonnabend <sonnabendch@gmail.com>
Date: Fri, 12 Jun 2026 13:11:36 +0200
Subject: [PATCH 4/6] Updating scripts

---
 .../analyze_gpu_benchmarks.py                 | 240 ++++++++++++++----
 .../gen_single_gpu_rtc_benchmark.sh           |  45 +---
 2 files changed, 194 insertions(+), 91 deletions(-)

diff --git a/prodtests/full-system-test/analyze_gpu_benchmarks.py b/prodtests/full-system-test/analyze_gpu_benchmarks.py
index 2320b16994dee..b3edbb9a12fd6 100644
--- a/prodtests/full-system-test/analyze_gpu_benchmarks.py
+++ b/prodtests/full-system-test/analyze_gpu_benchmarks.py
@@ -13,6 +13,13 @@
 except ImportError:
     SCIPY_AVAILABLE = False
 
+CYAN = "\033[96m"
+GREEN = "\033[92m"
+MAGENTA = "\033[95m"
+YELLOW = "\033[93m"
+RED = "\033[91m"
+BOLD = "\033[1m"
+RESET = "\033[0m"
 
 LINE_RE = re.compile(
     r"\[[^\]]*gpu-reconstruction[^\]]*\]:\s*"
@@ -82,70 +89,135 @@ def read_timeslice_durations(logfile: Path):
 
     return durations, starts, ends
 
-def warn_about_processing_downtime(starts, ends, tolerance_s=0.001):
-    RED = "\033[91m"
-    YELLOW = "\033[93m"
-    BOLD = "\033[1m"
-    RESET = "\033[0m"
+def format_ranges(values):
+    values = sorted(values)
+    if not values:
+        return "[]"
+
+    ranges = []
+    start = prev = values[0]
+
+    for value in values[1:]:
+        if value == prev + 1:
+            prev = value
+        else:
+            if start == prev:
+                ranges.append(f"{start}")
+            else:
+                ranges.append(f"{start}-{prev}")
+            start = prev = value
+
+    if start == prev:
+        ranges.append(f"{start}")
+    else:
+        ranges.append(f"{start}-{prev}")
 
-    common_timeslices = sorted(set(starts) & set(ends))
+    return "[" + ", ".join(ranges) + "]"
 
-    if len(common_timeslices) <= 4:
-        return set()
 
-    ignored_timeslices = set(common_timeslices[:2] + common_timeslices[-2:])
-    excluded_timeslices = set()
+def analyze_processing_sequences(starts, ends, tolerance_s=0.001, n_drop_edges=2):
+
+    complete_timeslices = sorted(set(starts) & set(ends))
 
-    for ts in common_timeslices:
-        next_ts = ts + 1
+    if len(complete_timeslices) <= 2 * n_drop_edges:
+        return set(), [], np.nan
+
+    used_timeslices = complete_timeslices[n_drop_edges:-n_drop_edges]
+    used_set = set(used_timeslices)
+
+    excluded_timeslices = set()
+    large_gap_boundaries = set()
 
-        if next_ts not in starts:
+    for ts, next_ts in zip(complete_timeslices[:-1], complete_timeslices[1:]):
+        if ts not in used_set or next_ts not in used_set:
             continue
 
-        # Do not warn if the boundary touches one of the first two or last two
-        # complete timeslices.
-        if ts in ignored_timeslices or next_ts in ignored_timeslices:
+        if next_ts != ts + 1:
+            print(
+                f"{RED}{BOLD}WARNING:{RESET} "
+                f"{RED}Missing timeslice(s) between {ts} and {next_ts}. "
+                f"Splitting processing sequence. "
+                f"Excluding timeslices {ts} and {next_ts} from calculation.{RESET}",
+                flush=True,
+            )
+            excluded_timeslices.update({ts, next_ts})
+            large_gap_boundaries.add((ts, next_ts))
             continue
 
         gap = starts[next_ts] - ends[ts]
 
         if gap > tolerance_s:
-            affected = {ts - 1, ts, next_ts, next_ts + 1}
-            affected = {
-                x for x in affected
-                if x in common_timeslices and x not in ignored_timeslices
-            }
-
             print(
                 f"{YELLOW}{BOLD}WARNING:{RESET} "
                 f"{YELLOW}Processing downtime detected between "
                 f"timeslice {ts} and {next_ts}: "
                 f"end[{ts}] -> start[{next_ts}] gap = {gap * 1000:.3f} ms. "
-                f"Excluding timeslices {sorted(affected)} from calculation.{RESET}",
+                f"Splitting processing sequence. "
+                f"Excluding timeslices {ts} and {next_ts} from calculation.{RESET}",
                 flush=True,
             )
-
-            excluded_timeslices.update(affected)
+            excluded_timeslices.update({ts, next_ts})
+            large_gap_boundaries.add((ts, next_ts))
 
         elif gap < -tolerance_s:
-            affected = {ts - 1, ts, next_ts, next_ts + 1}
-            affected = {
-                x for x in affected
-                if x in common_timeslices and x not in ignored_timeslices
-            }
-
             print(
                 f"{RED}{BOLD}WARNING:{RESET} "
                 f"{RED}Processing overlap or timestamp ordering issue between "
                 f"timeslice {ts} and {next_ts}: "
                 f"start[{next_ts}] is {-gap * 1000:.3f} ms before end[{ts}]. "
-                f"Excluding timeslices {sorted(affected)} from calculation.{RESET}",
+                f"Splitting processing sequence. "
+                f"Excluding timeslices {ts} and {next_ts} from calculation.{RESET}",
                 flush=True,
             )
+            excluded_timeslices.update({ts, next_ts})
+            large_gap_boundaries.add((ts, next_ts))
+
+    clean_timeslices = [
+        ts for ts in used_timeslices
+        if ts not in excluded_timeslices
+    ]
+
+    sequences = []
+    current_sequence = []
+
+    for ts in clean_timeslices:
+        if not current_sequence:
+            current_sequence = [ts]
+            continue
+
+        previous_ts = current_sequence[-1]
+
+        if (
+            ts == previous_ts + 1
+            and (previous_ts, ts) not in large_gap_boundaries
+        ):
+            current_sequence.append(ts)
+        else:
+            sequences.append(current_sequence)
+            current_sequence = [ts]
+
+    if current_sequence:
+        sequences.append(current_sequence)
+
+    total_wall_time = 0.0
+    total_timeslices = 0
 
-            excluded_timeslices.update(affected)
+    for sequence in sequences:
+        first_ts = sequence[0]
+        last_ts = sequence[-1]
 
-    return excluded_timeslices
+        # This includes small allowed gaps inside the sequence.
+        sequence_wall_time = ends[last_ts] - starts[first_ts]
+
+        total_wall_time += sequence_wall_time
+        total_timeslices += len(sequence)
+
+    if total_timeslices > 0:
+        wall_time_mean = total_wall_time / total_timeslices
+    else:
+        wall_time_mean = np.nan
+
+    return excluded_timeslices, sequences, wall_time_mean
 
 def fit_gaussian_to_histogram(values, bins):
     counts, edges = np.histogram(values, bins=bins)
@@ -213,6 +285,12 @@ def main():
         default=1.0,
         help="Allowed gap between end of timeslice n and start of timeslice n+1 in ms",
     )
+    parser.add_argument(
+        "--summary-output",
+        type=Path,
+        default=Path("gpu_reconstruction_summary.txt"),
+        help="Output text file for printed summary",
+    )
 
     args = parser.parse_args()
 
@@ -220,17 +298,26 @@ def main():
         args.logfile
     )
 
-    excluded_timeslices = warn_about_processing_downtime(
+    excluded_timeslices, processing_sequences, wall_time_mean = analyze_processing_sequences(
         starts_by_timeslice,
         ends_by_timeslice,
         tolerance_s=args.gap_tolerance_ms / 1000.0,
+        n_drop_edges=2,
     )
+
     if excluded_timeslices:
         print(
-            f"Excluded timeslices due to downtime/overlap: "
-            f"{sorted(excluded_timeslices)}"
+            f"{RED}{BOLD}Excluded timeslices due to large gaps/overlaps:{RESET} "
+            f"{RED}{sorted(excluded_timeslices)}{RESET}"
         )
 
+    print(
+        f"{RED}{BOLD}Continuous processing sequences used for wall-time average:{RESET} "
+        f"{RED}"
+        + ", ".join(format_ranges(seq) for seq in processing_sequences)
+        + f"{RESET}"
+    )
+
     if len(durations_by_timeslice) < 5:
         raise RuntimeError(
             f"Found only {len(durations_by_timeslice)} complete timeslices. "
@@ -260,7 +347,10 @@ def main():
     sample_mean = np.mean(values)
     sample_sigma = np.std(values, ddof=1)
 
-    avg_from_sum = np.mean(values)
+    if args.unit == "ms":
+        wall_time_mean_print = wall_time_mean * 1000.0
+    else:
+        wall_time_mean_print = wall_time_mean
 
     fit_result, counts, edges = fit_gaussian_to_histogram(values, args.bins)
 
@@ -301,23 +391,73 @@ def main():
     plt.tight_layout()
     plt.savefig(args.output, dpi=150)
 
-    print(f"Input file: {args.logfile}")
-    print(f"Complete timeslices found: {n_total}")
-    print(f"Timeslices used after dropping first/last two: {n_used}")
-    print(f"First used timeslice: {trimmed_timeslices[0]}")
-    print(f"Last used timeslice: {trimmed_timeslices[-1]}")
-    print(f"Average duration = sum(durations)/(processed timeslices - 4): {avg_from_sum:.6g} {unit_label}")
-    print(f"Sample mean: {sample_mean:.6g} {unit_label}")
-    print(f"Sample sigma: {sample_sigma:.6g} {unit_label}")
+    print(f"{BOLD}Input file:{RESET} {args.logfile}")
+    print(f"{CYAN}{BOLD}Complete timeslices found:{RESET} {n_total}")
+    print(f"{CYAN}{BOLD}Timeslices used after dropping first/last two:{RESET} {n_used}")
+    print(f"{CYAN}{BOLD}First used timeslice:{RESET} {trimmed_timeslices[0]}")
+    print(f"{CYAN}{BOLD}Last used timeslice:{RESET} {trimmed_timeslices[-1]}")
+
+    print(
+        f"{GREEN}{BOLD}Wall-time mean including allowed gaps:{RESET} "
+        f"{GREEN}{wall_time_mean_print:.6g} {unit_label}{RESET}"
+    )
+
+    print(
+        f"{MAGENTA}{BOLD}Individual duration sample mean:{RESET} "
+        f"{MAGENTA}{sample_mean:.6g} {unit_label}{RESET}"
+    )
+    print(
+        f"{MAGENTA}{BOLD}Individual duration sample sigma:{RESET} "
+        f"{MAGENTA}{sample_sigma:.6g} {unit_label}{RESET}"
+    )
 
     if fit_result is not None:
         _, fit_mean, fit_sigma = fit_result
-        print(f"Gaussian fit mean: {fit_mean:.6g} {unit_label}")
-        print(f"Gaussian fit sigma: {fit_sigma:.6g} {unit_label}")
+        print(
+            f"{YELLOW}{BOLD}Gaussian fit mean:{RESET} "
+            f"{YELLOW}{fit_mean:.6g} {unit_label}{RESET}"
+        )
+        print(
+            f"{YELLOW}{BOLD}Gaussian fit sigma:{RESET} "
+            f"{YELLOW}{fit_sigma:.6g} {unit_label}{RESET}"
+        )
     else:
-        print("Gaussian fit failed or scipy is unavailable.")
+        print(f"{RED}{BOLD}Gaussian fit failed or scipy is unavailable.{RESET}")
+
+    if args.summary_output:
+
+        def save_summary_output(output_file: Path, lines):
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+
+            with output_file.open("w") as f:
+                for line in lines:
+                    f.write(line + "\n")
+
+            print(f"Saved summary output to: {output_file}")
+
+        summary_lines = [
+            f"Input file: {args.logfile}",
+            f"Complete timeslices found: {n_total}",
+            f"Timeslices used after dropping first/last two: {n_used}",
+            f"First used timeslice: {trimmed_timeslices[0]}",
+            f"Last used timeslice: {trimmed_timeslices[-1]}",
+            f"Wall-time mean including allowed gaps: {wall_time_mean_print:.6g} {unit_label}",
+            f"Individual duration sample mean: {sample_mean:.6g} {unit_label}",
+            f"Individual duration sample sigma: {sample_sigma:.6g} {unit_label}",
+        ]
+
+        if fit_result is not None:
+            _, fit_mean, fit_sigma = fit_result
+            summary_lines.extend(
+                [
+                    f"Gaussian fit mean: {fit_mean:.6g} {unit_label}",
+                    f"Gaussian fit sigma: {fit_sigma:.6g} {unit_label}",
+                ]
+            )
+        else:
+            summary_lines.append("Gaussian fit failed or scipy is unavailable.")
 
-    print(f"Saved plot to: {args.output}")
+        save_summary_output(args.summary_output, summary_lines)
 
 
 if __name__ == "__main__":
diff --git a/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh b/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
index e48d779f9e294..21c274d8a5ea6 100755
--- a/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
+++ b/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
@@ -8,13 +8,7 @@ set -euo pipefail
 
 rm -rf /dev/shm/fmq*
 
-# ----------------------------------------------------------------------------------------------------------------------
-# Locate original workflow script. Keep the original untouched.
-
-: "${GEN_TOPO_MYDIR:=$(dirname "$(realpath "$0")")}"
-: "${O2_DPL_WORKFLOW:=$GEN_TOPO_MYDIR/dpl-workflow.sh}"
-: "${GEN_TOPO_SOURCE_SCRIPT:=$O2_DPL_WORKFLOW}"   # backward compatibility only
-O2_DPL_WORKFLOW="$GEN_TOPO_SOURCE_SCRIPT"
+: "${O2_DPL_WORKFLOW:=$O2_ROOT/prodtests/full-system-test/dpl-workflow.sh}"
 
 if [[ ! -f "$O2_DPL_WORKFLOW" ]]; then
   echo "FATAL: dpl workflow script does not exist: $O2_DPL_WORKFLOW" >&2
@@ -22,23 +16,15 @@ if [[ ! -f "$O2_DPL_WORKFLOW" ]]; then
   exit 1
 fi
 
-# Helper lookup must remain compatible with the original script directory.
-export GEN_TOPO_MYDIR="$(dirname "$(realpath "$O2_DPL_WORKFLOW")")"
-
-# ----------------------------------------------------------------------------------------------------------------------
-# Match the normal FST startup environment.
-
 # ----------------------------------------------------------------------------------------------------------------------
 # Benchmark defaults. All can be overridden by exporting variables before calling this script.
 
 export DPL_REPORT_PROCESSING="${DPL_REPORT_PROCESSING:-1}"
 
-export FST_TMUX_NO_EPN="${FST_TMUX_NO_EPN:-1}"
+# export FST_TMUX_NO_EPN="${FST_TMUX_NO_EPN:-1}"
 export WORKFLOW_PARAMETERS="${WORKFLOW_PARAMETERS:-GPU,CTF}"
 export GPUTYPE="${GPUTYPE:-CUDA}"
 export NGPUS=1
-export NUMAGPUIDS=1
-export NUMAID="${NUMAID:-0}"
 
 export O2_GPU_DOUBLE_PIPELINE="${O2_GPU_DOUBLE_PIPELINE:-1}"
 export O2_GPU_RTC="${O2_GPU_RTC:-1}"
@@ -48,7 +34,6 @@ export SYNCMODE="${SYNCMODE:-1}"
 export SYNCRAWMODE="${SYNCRAWMODE:-0}"
 
 export TIMEFRAME_RATE_LIMIT="${TIMEFRAME_RATE_LIMIT:-5}"
-export GEN_TOPO_NO_TF_RATE_UPSCALING="${GEN_TOPO_NO_TF_RATE_UPSCALING:-1}"
 
 export DISABLE_ROOT_OUTPUT="${DISABLE_ROOT_OUTPUT:-1}"
 
@@ -68,17 +53,6 @@ if [[ -f "$PWD/local_env.sh" ]]; then
   source "$PWD/local_env.sh"
 fi
 
-export ALICE_O2_FST="${ALICE_O2_FST:-1}"
-
-if [[ -f "$GEN_TOPO_MYDIR/setenv.sh" ]]; then
-  source "$GEN_TOPO_MYDIR/setenv.sh" || {
-    echo "FATAL: setenv.sh failed: $GEN_TOPO_MYDIR/setenv.sh" >&2
-    exit 1
-  }
-else
-  echo "WARNING: setenv.sh not found: $GEN_TOPO_MYDIR/setenv.sh" >&2
-fi
-
 echo "# Alien/JAliEn environment check:"
 echo "#   JALIEN_TOKEN_CERT=${JALIEN_TOKEN_CERT:-}"
 echo "#   JALIEN_TOKEN_KEY=${JALIEN_TOKEN_KEY:-}"
@@ -162,7 +136,6 @@ trap cleanup_rundir EXIT
 # Let O2/core dumps land in the benchmark run directory, not in the original working directory.
 export CORE_DUMP_DIR="${CORE_DUMP_DIR:-$RUNDIR}"
 export O2_CORE_DUMP_DIR="${O2_CORE_DUMP_DIR:-$RUNDIR}"
-export FAIRMQ_SHM_MONITOR_CONFIG="${FAIRMQ_SHM_MONITOR_CONFIG:-}"
 
 # Avoid copying input files unless the caller explicitly requests a copy command.
 if [[ "${BENCH_DISABLE_INPUT_COPY:-1}" == "1" ]]; then
@@ -191,7 +164,7 @@ fi
 
 # ROCm library injection is only useful for HIP runs. Keep it off by default for CUDA/NVIDIA containers,
 # because mixed AMD/NVIDIA hosts can otherwise leak ROCm libraries into LD_LIBRARY_PATH.
-if [[ "${GPUTYPE:-}" == "HIP" && "0$BENCH_AUTO_ROCM_LIBS" == "01" ]]; then
+if [[ "${GPUTYPE:-}" == "HIP" && $BENCH_AUTO_ROCM_LIBS == 1 ]]; then
   if [[ -n "${ROCM_PATH:-}" ]]; then
     prepend_ld_path "$ROCM_PATH/lib64"
     prepend_ld_path "$ROCM_PATH/lib"
@@ -281,7 +254,7 @@ echo "# single-GPU RTC benchmark"
 echo "# source script: $O2_DPL_WORKFLOW"
 echo "# output dir:    $OUTDIR"
 echo "# run dir:       $RUNDIR"
-echo "# NGPUS=$NGPUS NUMAGPUIDS=$NUMAGPUIDS NUMAID=$NUMAID GPUTYPE=$GPUTYPE"
+echo "# NGPUS=$NGPUS GPUTYPE=$GPUTYPE"
 echo "# O2_GPU_DOUBLE_PIPELINE=$O2_GPU_DOUBLE_PIPELINE O2_GPU_RTC=$O2_GPU_RTC"
 echo "# CTFINPUT=$CTFINPUT RAWTFINPUT=$RAWTFINPUT DIGITINPUT=$DIGITINPUT EXTINPUT=$EXTINPUT NTIMEFRAMES=$NTIMEFRAMES TFLOOP=$TFLOOP"
 echo "# FILEWORKDIR=${FILEWORKDIR:-} INPUT_FILE_LIST=${INPUT_FILE_LIST:-}"
@@ -312,16 +285,6 @@ if [[ "$RUN_BENCHMARK" == "1" ]]; then
   status=$?
   set -e
 
-  # {
-  #   echo "tag,host,date,ngpus,numagpuids,double_pipeline,rtc,ntf,tfloop,elapsed_seconds,max_rss_kb,exit_status"
-  #   awk '
-  #     /Elapsed \(wall clock\) time/ {elapsed=$NF}
-  #     /Maximum resident set size/ {rss=$NF}
-  #     /Exit status/ {status=$NF}
-  #     END {print ENVIRON["BENCH_TAG"] "," ENVIRON["HOSTNAME"] "," strftime("%FT%T%z") "," ENVIRON["NGPUS"] "," ENVIRON["NUMAGPUIDS"] "," ENVIRON["O2_GPU_DOUBLE_PIPELINE"] "," ENVIRON["O2_GPU_RTC"] "," ENVIRON["NTIMEFRAMES"] "," ENVIRON["TFLOOP"] "," elapsed "," rss "," status}' "$log"
-  # } > "$OUTDIR/summary_${BENCH_TAG}_${BENCH_STAMP}.csv"
-
-  # echo "# Summary: $OUTDIR/summary_${BENCH_TAG}_${BENCH_STAMP}.csv"
   echo "# Full log: $log"
 
   # --------------------------------------------------------------------------------------------------------------------

From 43e244bfc3ff8a44e79e2398e972bbece3ca7b99 Mon Sep 17 00:00:00 2001
From: Christian Sonnabend <sonnabendch@gmail.com>
Date: Fri, 12 Jun 2026 14:36:01 +0200
Subject: [PATCH 5/6] Update env variables

---
 .../gen_single_gpu_rtc_benchmark.sh           | 49 +------------------
 1 file changed, 2 insertions(+), 47 deletions(-)

diff --git a/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh b/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
index 21c274d8a5ea6..1ab68f653a889 100755
--- a/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
+++ b/prodtests/full-system-test/gen_single_gpu_rtc_benchmark.sh
@@ -20,39 +20,24 @@ fi
 # Benchmark defaults. All can be overridden by exporting variables before calling this script.
 
 export DPL_REPORT_PROCESSING="${DPL_REPORT_PROCESSING:-1}"
-
-# export FST_TMUX_NO_EPN="${FST_TMUX_NO_EPN:-1}"
 export WORKFLOW_PARAMETERS="${WORKFLOW_PARAMETERS:-GPU,CTF}"
 export GPUTYPE="${GPUTYPE:-CUDA}"
 export NGPUS=1
-
 export O2_GPU_DOUBLE_PIPELINE="${O2_GPU_DOUBLE_PIPELINE:-1}"
 export O2_GPU_RTC="${O2_GPU_RTC:-1}"
-
-export EPNSYNCMODE="${EPNSYNCMODE:-0}"
 export SYNCMODE="${SYNCMODE:-1}"
-export SYNCRAWMODE="${SYNCRAWMODE:-0}"
-
-export TIMEFRAME_RATE_LIMIT="${TIMEFRAME_RATE_LIMIT:-5}"
-
 export DISABLE_ROOT_OUTPUT="${DISABLE_ROOT_OUTPUT:-1}"
 
 # Double pipeline requires zsraw input. Therefore default to raw TF input, not CTF.
-export CTFINPUT="${CTFINPUT:-0}"
 export RAWTFINPUT="${RAWTFINPUT:-1}"
-export DIGITINPUT="${DIGITINPUT:-0}"
-export EXTINPUT="${EXTINPUT:-0}"
 
 export NTIMEFRAMES="${NTIMEFRAMES:--1}"
 export TFLOOP="${TFLOOP:-100}"
 export TFDELAY="${TFDELAY:-0}"
+export TIMEFRAME_RATE_LIMIT="${TIMEFRAME_RATE_LIMIT:-5}"
 
 export RUN_BENCHMARK="${RUN_BENCHMARK:-0}"
 
-if [[ -f "$PWD/local_env.sh" ]]; then
-  source "$PWD/local_env.sh"
-fi
-
 echo "# Alien/JAliEn environment check:"
 echo "#   JALIEN_TOKEN_CERT=${JALIEN_TOKEN_CERT:-}"
 echo "#   JALIEN_TOKEN_KEY=${JALIEN_TOKEN_KEY:-}"
@@ -178,36 +163,6 @@ if [[ -n "${BENCH_EXTRA_LD_LIBRARY_PATH:-}" ]]; then
   export LD_LIBRARY_PATH="$BENCH_EXTRA_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"
 fi
 
-# Fail early for the unsupported combination instead of letting o2-gpu-reco-workflow crash later.
-if [[ "${O2_GPU_DOUBLE_PIPELINE:-0}" == "1" ]]; then
-  if [[ "${CTFINPUT:-0}" == "1" ]]; then
-    echo "FATAL: O2_GPU_DOUBLE_PIPELINE=1 is incompatible with CTFINPUT=1 in dpl-workflow.sh." >&2
-    echo "Double pipeline requires o2-gpu-reco-workflow --input-type=zsraw." >&2
-    echo "Use RAWTFINPUT=1 or rawAll.cfg input, or set O2_GPU_DOUBLE_PIPELINE=0 for CTF benchmarking." >&2
-    exit 1
-  fi
-  if [[ "${DIGITINPUT:-0}" == "1" ]]; then
-    echo "FATAL: O2_GPU_DOUBLE_PIPELINE=1 is not suitable for DIGITINPUT=1 in dpl-workflow.sh." >&2
-    echo "Digit input uses zsonthefly and is restricted to NTIMEFRAMES=1." >&2
-    exit 1
-  fi
-fi
-
-# Input checks with clearer messages.
-if [[ "${RAWTFINPUT:-0}" == "1" ]]; then
-  if [[ -z "${FILEWORKDIR:-}" && -z "${INPUT_FILE_LIST:-}" ]]; then
-    echo "FATAL: RAWTFINPUT=1 but neither FILEWORKDIR nor INPUT_FILE_LIST is set." >&2
-    echo "Set FILEWORKDIR=/path/to/raw_tf_dir or INPUT_FILE_LIST=/path/to/o2_*.tf" >&2
-    exit 1
-  fi
-  if [[ -z "${INPUT_FILE_LIST:-}" ]] && ! ls "${FILEWORKDIR}"/o2_*.tf >/dev/null 2>&1; then
-    echo "FATAL: RAWTFINPUT=1 but no raw TF file was found." >&2
-    echo "Looked for: ${FILEWORKDIR}/o2_*.tf" >&2
-    echo "Set FILEWORKDIR=/path/to/raw_tf_dir or INPUT_FILE_LIST=/path/to/o2_*.tf" >&2
-    exit 1
-  fi
-fi
-
 # Check CUDA runtime/device visibility before starting the full workflow.
 if [[ "$GPUTYPE" == "CUDA" ]]; then
   if ! command -v nvidia-smi >/dev/null 2>&1; then
@@ -256,7 +211,7 @@ echo "# output dir:    $OUTDIR"
 echo "# run dir:       $RUNDIR"
 echo "# NGPUS=$NGPUS GPUTYPE=$GPUTYPE"
 echo "# O2_GPU_DOUBLE_PIPELINE=$O2_GPU_DOUBLE_PIPELINE O2_GPU_RTC=$O2_GPU_RTC"
-echo "# CTFINPUT=$CTFINPUT RAWTFINPUT=$RAWTFINPUT DIGITINPUT=$DIGITINPUT EXTINPUT=$EXTINPUT NTIMEFRAMES=$NTIMEFRAMES TFLOOP=$TFLOOP"
+echo "# NTIMEFRAMES=$NTIMEFRAMES TFLOOP=$TFLOOP"
 echo "# FILEWORKDIR=${FILEWORKDIR:-} INPUT_FILE_LIST=${INPUT_FILE_LIST:-}"
 echo "# LD_LIBRARY_PATH font-lib workaround: BENCH_USE_SYSTEM_FONT_LIBS=$BENCH_USE_SYSTEM_FONT_LIBS"
 echo "# ROCm library auto-detect: BENCH_AUTO_ROCM_LIBS=$BENCH_AUTO_ROCM_LIBS (active only when GPUTYPE=HIP)"

From 6353bf6c0db100e87575a05f2d89ba02dced93af Mon Sep 17 00:00:00 2001
From: Christian Sonnabend <sonnabendch@gmail.com>
Date: Fri, 12 Jun 2026 20:18:17 +0200
Subject: [PATCH 6/6] Adding microsecond logging for dpl-workflow.sh

---
 prodtests/full-system-test/analyze_gpu_benchmarks.py |  2 +-
 prodtests/full-system-test/dpl-workflow.sh           | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/prodtests/full-system-test/analyze_gpu_benchmarks.py b/prodtests/full-system-test/analyze_gpu_benchmarks.py
index b3edbb9a12fd6..79c8ee43df8ab 100644
--- a/prodtests/full-system-test/analyze_gpu_benchmarks.py
+++ b/prodtests/full-system-test/analyze_gpu_benchmarks.py
@@ -282,7 +282,7 @@ def main():
     parser.add_argument(
         "--gap-tolerance-ms",
         type=float,
-        default=1.0,
+        default=50.0,
         help="Allowed gap between end of timeslice n and start of timeslice n+1 in ms",
     )
     parser.add_argument(
diff --git a/prodtests/full-system-test/dpl-workflow.sh b/prodtests/full-system-test/dpl-workflow.sh
index 69ffcaf9ff378..5525cb8bddb99 100755
--- a/prodtests/full-system-test/dpl-workflow.sh
+++ b/prodtests/full-system-test/dpl-workflow.sh
@@ -163,6 +163,12 @@ EVE_OPT=" --jsons-folder $EDJSONS_DIR"
 : ${MFTSTAGGERED:=}
 : ${PROXY_INSPEC:=}
 
+if [[ $MICROSEC_REPORTING == 1 ]]; then
+  MSLOG="--log-timestamp-us"
+else
+  MSLOG=
+fi
+
 [[ -z $ALPIDE_ERR_DUMPS ]] && [[ $EPNSYNCMODE == 1 && $RUNTYPE == "PHYSICS" ]] && ALPIDE_ERR_DUMPS=1 || ALPIDE_ERR_DUMPS=0
 
 [[ "0$DISABLE_ROOT_OUTPUT" == "00" ]] && DISABLE_ROOT_OUTPUT=
@@ -620,14 +626,14 @@ has_detector_gpu ITS && GPU_OUTPUT+=",its-tracks"
 # Common reconstruction workflows
 (has_detector_reco ITS && ! has_detector_gpu ITS) && ! has_detector_from_global_reader ITS && add_W o2-its-reco-workflow "$ITS_CONFIG $ITS_STAGGERED $DISABLE_MC ${DISABLE_DIGIT_CLUSTER_INPUT:-} $DISABLE_ROOT_OUTPUT --pipeline $(get_N its-tracker ITS REST 1 ITSTRK),$(get_N its-clusterer ITS REST 1 ITSCL)" "$ITS_CONFIG_KEY;$ITSMFT_STROBES;$ITSEXTRAERR"
 [[ ${DISABLE_DIGIT_CLUSTER_INPUT:-} =~ "--digits-from-upstream" ]]  && has_detector_gpu ITS && ! has_detector_from_global_reader ITS && add_W o2-its-reco-workflow "--disable-tracking ${DISABLE_DIGIT_CLUSTER_INPUT:-} $ITS_STAGGERED $DISABLE_MC $DISABLE_ROOT_OUTPUT --pipeline $(get_N its-clusterer ITS REST 1 ITSCL)" "$ITS_CONFIG_KEY;$ITSMFT_STROBES;$ITSEXTRAERR"
-(has_detector_reco TPC || has_detector_ctf TPC) && ! has_detector_from_global_reader TPC && add_W o2-gpu-reco-workflow "--gpu-reconstruction \"$GPU_CONFIG_SELF\" --input-type=$GPU_INPUT $DISABLE_MC --output-type $GPU_OUTPUT $([[ $TPC_CORR_OPT == *--disable-ctp-lumi-request* ]] && echo --disable-ctp-lumi-request) $ITS_STAGGERED --pipeline gpu-reconstruction:${N_TPCTRK:-1},gpu-reconstruction-prepare:${N_TPCTRK:-1} $GPU_CONFIG" "GPU_global.deviceType=$GPUTYPE;GPU_proc.debugLevel=0;$GPU_CONFIG_KEY;$TRACKTUNETPCINNER;"
+(has_detector_reco TPC || has_detector_ctf TPC) && ! has_detector_from_global_reader TPC && add_W o2-gpu-reco-workflow "--gpu-reconstruction \"$GPU_CONFIG_SELF\" $MSLOG --input-type=$GPU_INPUT $DISABLE_MC --output-type $GPU_OUTPUT $([[ $TPC_CORR_OPT == *--disable-ctp-lumi-request* ]] && echo --disable-ctp-lumi-request) $ITS_STAGGERED --pipeline gpu-reconstruction:${N_TPCTRK:-1},gpu-reconstruction-prepare:${N_TPCTRK:-1} $GPU_CONFIG" "GPU_global.deviceType=$GPUTYPE;GPU_proc.debugLevel=0;$GPU_CONFIG_KEY;$TRACKTUNETPCINNER;"
 (has_detector_reco TOF || has_detector_ctf TOF) && ! has_detector_from_global_reader TOF && add_W o2-tof-reco-workflow "$TOF_CONFIG --input-type $TOF_INPUT --output-type $TOF_OUTPUT $DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N tof-compressed-decoder TOF RAW 1),$(get_N TOFClusterer TOF REST 1)"
 has_detector_reco FT0 && ! has_detector_from_global_reader FT0 && add_W o2-ft0-reco-workflow "$DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N ft0-reconstructor FT0 REST 1)"
 has_detector_reco TRD && ! has_detector_from_global_reader TRD && add_W o2-trd-tracklet-transformer "--disable-irframe-reader $DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC $TRD_FILTER_CONFIG --pipeline $(get_N TRDTRACKLETTRANSFORMER TRD REST 1 TRDTRKTRANS)"
 has_detectors_reco ITS TPC && ! has_detector_from_global_reader_tracks ITS-TPC && has_detector_matching ITSTPC && add_W o2-tpcits-match-workflow "$DISABLE_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC $ITS_STAGGERED $SEND_ITSTPC_DTGL $TPC_CORR_OPT --nthreads $ITSTPC_THREADS --pipeline $(get_N itstpc-track-matcher MATCH REST $ITSTPC_THREADS TPCITS)" "$ITSTPC_CONFIG_KEY;$INTERACTION_TAG_CONFIG_KEY;$ITSMFT_STROBES;$ITSEXTRAERR;"
 has_detector_reco TRD && [[ -n "$TRD_SOURCES" ]] && ! has_detector_from_global_reader_tracks "$(echo "$TRD_SOURCES" | cut -d',' -f1)-TRD" && add_W o2-trd-global-tracking "$DISABLE_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC $TRD_CONFIG $TRD_FILTER_CONFIG $TPC_CORR_OPT --track-sources $TRD_SOURCES --pipeline $(get_N trd-globaltracking_TPC_ITS-TPC_ TRD REST 1 TRDTRK),$(get_N trd-globaltracking_TPC_FT0_ITS-TPC_ TRD REST 1 TRDTRK),$(get_N trd-globaltracking_TPC_FT0_ITS-TPC_CTP_ TRD REST 1 TRDTRK)" "$TRD_CONFIG_KEY;$INTERACTION_TAG_CONFIG_KEY;$ITSMFT_STROBES;$ITSEXTRAERR;"
 has_detector_reco TOF && [[ -n "$TOF_SOURCES" ]] && ! has_detector_from_global_reader_tracks "$(echo "$TOF_SOURCES" | cut -d',' -f1)-TOF" && add_W o2-tof-matcher-workflow "$TOF_MATCH_OPT $DISABLE_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC $TPC_CORR_OPT ${TOFMATCH_THREADS:+--tof-lanes ${TOFMATCH_THREADS}} --track-sources $TOF_SOURCES --pipeline $(get_N tof-matcher TOF REST 1 TOFMATCH)" "$ITSMFT_STROBES;$ITSEXTRAERR;$INTERACTION_TAG_CONFIG_KEY"
-has_detectors TPC && [[ -z "$DISABLE_ROOT_OUTPUT" && "${SKIP_TPC_CLUSTERSTRACKS_OUTPUT:-}" != 1 ]] && ! has_detector_from_global_reader TPC && add_W o2-tpc-reco-workflow "--input-type pass-through --output-type clusters,tpc-triggers,tracks,send-clusters-per-sector $DISABLE_MC"
+has_detectors TPC && [[ -z "$DISABLE_ROOT_OUTPUT" && "${SKIP_TPC_CLUSTERSTRACKS_OUTPUT:-}" != 1 ]] && ! has_detector_from_global_reader TPC && add_W o2-tpc-reco-workflow "--input-type pass-through --output-type clusters,tpc-triggers,tracks,send-clusters-per-sector $DISABLE_MC $MSLOG"
 
 # ---------------------------------------------------------------------------------------------------------------------
 # Reconstruction workflows normally active only in async mode ($LIST_OF_ASYNC_RECO_STEPS), but can be forced via $WORKFLOW_EXTRA_PROCESSING_STEPS