Add scoped event-time director models

CODES-compile-instructions.sh

@@ -37,6 +37,17 @@ else
     echo "Using existing ross checkout: $(realpath ross)"
 fi
 
+
+if [ "$torch_enable" = 1 ]; then
+    make_args_codes=(
+        "${make_args_codes[@]}"
+    )
+else
+    make_args_codes=(
+        "${make_args_codes[@]}"
+    )
+fi
+
 if [ $swm_enable = 1 ]; then
     if [ ! -d argobots/.git ]; then
     git clone https://github.com/pmodels/argobots --depth=1
@@ -192,53 +203,41 @@ fi
 
 
 
-# Make system pkg-config metadata visible even when Conda's pkg-config is active.
-# This is needed for libzmq.pc on systems where ZeroMQ is installed through the OS
-# but the active Conda environment's pkg-config only searches Conda pkgconfig dirs.
-if ! pkg-config --exists libzmq 2>/dev/null; then
-    for pcdir in \
-        /usr/lib/x86_64-linux-gnu/pkgconfig \
-        /usr/lib64/pkgconfig \
-        /usr/lib/pkgconfig \
-        /usr/local/lib/pkgconfig \
-        /usr/local/lib64/pkgconfig \
-        /opt/homebrew/lib/pkgconfig \
-        /usr/share/pkgconfig
-    do
-        if [ -d "$pcdir" ]; then
-            export PKG_CONFIG_PATH="$pcdir:${PKG_CONFIG_PATH:-}"
-        fi
-    done
-fi
+if [ "$torch_enable" = 1 ]; then
+    # Make system pkg-config metadata visible even when Conda's pkg-config is active.
+    # This is needed for libzmq.pc on systems where ZeroMQ is installed through the OS
+    # but the active Conda environment's pkg-config only searches Conda pkgconfig dirs.
+    if ! pkg-config --exists libzmq 2>/dev/null; then
+        for pcdir in \
+            /usr/lib/x86_64-linux-gnu/pkgconfig \
+            /usr/lib64/pkgconfig \
+            /usr/lib/pkgconfig \
+            /usr/local/lib/pkgconfig \
+            /usr/local/lib64/pkgconfig \
+            /opt/homebrew/lib/pkgconfig \
+            /usr/share/pkgconfig
+        do
+            if [ -d "$pcdir" ]; then
+                export PKG_CONFIG_PATH="$pcdir:${PKG_CONFIG_PATH:-}"
+            fi
+        done
+    fi
+
+    if ! pkg-config --exists libzmq 2>/dev/null; then
+        echo "WARNING: pkg-config still cannot find libzmq.pc." >&2
+        echo "         If ZMQML requester support fails to build, install the ZeroMQ development package" >&2
+        echo "         or set PKG_CONFIG_PATH to the directory containing libzmq.pc." >&2
+    fi
 
-if ! pkg-config --exists libzmq 2>/dev/null; then
-    echo "WARNING: pkg-config still cannot find libzmq.pc." >&2
-    echo "         If ZMQML fails to build, install the ZeroMQ development package" >&2
-    echo "         or set PKG_CONFIG_PATH to the directory containing libzmq.pc." >&2
+    # Build local ZMQML requester library required by director-client.C.
+    pushd codes/src/surrogate/zmqml
+    make clean
+    make
+    test -f libzmqmlrequester.so
+    test -f zmqmlrequester.h
+    popd
 fi
 
-# Build local ZMQML requester library required by director-client.C
-pushd codes/src/surrogate/zmqml
-make clean
-make
-test -f libzmqmlrequester.so
-test -f zmqmlrequester.h
-popd
-
-# Make imported zmqmlrequester target visible to doc/example and tests.
-python3 - <<'INNERPY'
-from pathlib import Path
-cm = Path("codes/src/CMakeLists.txt")
-text = cm.read_text()
-old = "add_library(zmqmlrequester SHARED IMPORTED )"
-new = "add_library(zmqmlrequester SHARED IMPORTED GLOBAL)"
-if old in text:
-    cm.write_text(text.replace(old, new))
-elif new in text:
-    pass
-else:
-    raise SystemExit("Could not find zmqmlrequester imported target line in codes/src/CMakeLists.txt")
-INNERPY
 
 mkdir -p codes/build
 pushd codes/build
@@ -368,10 +367,8 @@ make_args_codes=(
     -DCMAKE_USE_WIN32_THREADS_INIT=0
     -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON
     -DCMAKE_INSTALL_PREFIX="$(realpath bin)"
-    -DZMQML_BUILD_PATH="$(realpath "$CUR_DIR/codes/src/surrogate/zmqml")"
-    -DZeroMQ_INCLUDE_DIR=/usr/include
-    -DZeroMQ_LIBRARY=/usr/lib/x86_64-linux-gnu/libzmq.so
 )
+
 if [ $swm_enable = 1 ]; then
     make_args_codes=(
         "${make_args_codes[@]}"
@@ -390,6 +387,10 @@ if [ "$torch_enable" = 1 ]; then
         "${make_args_codes[@]}"
         -DUSE_TORCH=true
         -DTorch_DIR="${torch_dir}"
+        -DUSE_ZMQML=true
+        -DZMQML_BUILD_PATH="$(realpath "$CUR_DIR/codes/src/surrogate/zmqml")"
+        -DZeroMQ_INCLUDE_DIR=/usr/include
+        -DZeroMQ_LIBRARY=/usr/lib/x86_64-linux-gnu/libzmq.so
     )
 
     if [ -n "${CUDA_HOME:-}" ]; then
@@ -412,7 +413,10 @@ if [ "$torch_enable" = 1 ]; then
         )
     fi
 else
-    make_args_codes=("${make_args_codes[@]}" -DUSE_TORCH=false)
+    make_args_codes=(
+        "${make_args_codes[@]}"
+        -DUSE_TORCH=false
+    )
 fi
 
 cmake .. "${make_args_codes[@]}"

README.md

@@ -139,6 +139,7 @@ This repo uses [clang-format](https://clang.llvm.org/docs/ClangFormat.html) to k
 - **Emacs:** see [clang-format.el](https://clang.llvm.org/docs/ClangFormat.html#emacs-integration).
 
 To reformat a file manually: `clang-format -i path/to/file.c`. CI runs `clang-format --dry-run --Werror` on every PR and rejects any drift, so PRs with unformatted code don't merge.
+Note: The CI uses clang-format major release version 20, so you should format your files with that version.
 
 ### Determinism
 

codes/surrogate/director-client.h

@@ -125,8 +125,7 @@ extern "C" {
 
 
 extern void director_lp_register_model(const char*);
-
-
+extern void director_record_external_zmq_latency(double processing_sec, double total_sec);
 /*
 extern void director_parse_args(char *args, int **args_array, int *length);
 static void director_issue_codes_event(director_state * s, tw_lpid nw_lpid, int dir_registered_event_type, tw_stime ts, tw_lp* lp);
@@ -142,5 +141,4 @@ extern void dir_test_finalize(director_state* s, tw_lp* lp);
 #ifdef __cplusplus
 }
 #endif
-
 #endif

doc/example/kb.dfdally-72-zeromq-director.conf.in

@@ -23,19 +23,19 @@ LPGROUPS
 
 DIRECTOR
 {
-    start_iter="${DIRECTOR_START_ITER}";
-    end_iter="${DIRECTOR_END_ITER}";
+    start_iter="${START_ITER}";
+    end_iter="${END_ITER}";
 
     # Optional one-shot pause/retrain/resume pipeline.
     # First implementation is intended for --synch=1.
-    retrain_enabled="${DIRECTOR_RETRAIN_ENABLED}";
-    retrain_iter="${DIRECTOR_RETRAIN_ITER}";
-    retrain_save_path="${DIRECTOR_RETRAIN_SAVE_PATH}";
+    retrain_enabled="${RETRAIN_ENABLED}";
+    retrain_iter="${RETRAIN_ITER}";
+    retrain_save_path="${RETRAIN_SAVE_PATH}";
 
     # Optional second surrogate window after retraining.
-    second_surrogate_enabled="${DIRECTOR_SECOND_SURROGATE_ENABLED}";
-    second_start_iter="${DIRECTOR_SECOND_START_ITER}";
-    second_end_iter="${DIRECTOR_SECOND_END_ITER}";
+    second_surrogate_enabled="${SECOND_SURROGATE_ENABLED}";
+    second_start_iter="${SECOND_START_ITER}";
+    second_end_iter="${SECOND_END_ITER}";
 
     # Common modes:
     #

src/CMakeLists.txt

@@ -1,3 +1,4 @@
+option(USE_ZMQML "Enable ZeroMQ ML requester support" OFF)
 cmake_print_variables(CMAKE_CURRENT_SOURCE_DIR)
 
 find_package(FLEX REQUIRED)
@@ -124,68 +125,38 @@ if(USE_TORCH)
     list(APPEND LIBS_TO_LINK ${TORCH_LIBRARIES})
 endif()
 
-# ZMQML / director-client (opt-in). When USE_ZMQML=ON, callers must
-# point ZMQML_BUILD_PATH at a directory containing libzmqmlrequester.so
-# (build it via src/surrogate/zmqml/Makefile, or set ZMQML_BUILD_PATH to
-# wherever you installed it). When OFF (the default), CODES builds with
-# no surrogate/director-client linkage; configs that reference
-# "dir-nw-lp" will fail at runtime because the LP type isn't registered.
-option(USE_ZMQML "Build the director-client + zmqml surrogate integration" OFF)
+# ZMQML requester support
 if(USE_ZMQML)
-    if(NOT ZMQML_BUILD_PATH)
-        message(FATAL_ERROR
-            "USE_ZMQML=ON but ZMQML_BUILD_PATH is unset.\n"
-            "Build src/surrogate/zmqml/libzmqmlrequester.so first, then "
-            "reconfigure with -DZMQML_BUILD_PATH=<dir containing the .so>.")
-    endif()
     list(APPEND SRCS surrogate/director-client.C)
-endif()
 
-add_library(codes STATIC ${SRCS})
-
-list(APPEND LIBS_TO_LINK ${MPI_C_LIBRARIES})
-target_include_directories(codes INTERFACE ${MPI_C_INCLUDE_PATH})
+    if(NOT DEFINED ZMQML_BUILD_PATH)
+        message(FATAL_ERROR "USE_ZMQML is ON, but ZMQML_BUILD_PATH is not defined.")
+    endif()
 
-# set(LIBS_TO_LINK
-#     PkgConfig::ROSS
-#     ${DUMPI_LIB}
-#     PkgConfig::ARGOBOTS
-#     PkgConfig::SWM
-# )
+    if(NOT EXISTS "${ZMQML_BUILD_PATH}/libzmqmlrequester.so")
+        message(FATAL_ERROR "USE_ZMQML is ON, but ${ZMQML_BUILD_PATH}/libzmqmlrequester.so does not exist. Re-run CODES-compile-instructions.sh so the local requester library is built before configuring CODES.")
+    endif()
 
-#LINK DUMPI
-# target_link_libraries(codes PUBLIC ${DUPMI_LIB})
-if(USE_DUMPI)
-    target_include_directories(codes PUBLIC ${DUMPI_INCLUDE})
-endif()
+    pkg_check_modules(PC_ZeroMQ QUIET zmq)
+    find_path(ZeroMQ_INCLUDE_DIR NAMES zmq.hpp PATHS ${PC_ZeroMQ_INCLUDE_DIRS})
+    find_library(ZeroMQ_LIBRARY NAMES zmq PATHS ${PC_ZeroMQ_LIBRARY_DIRS})
 
-#LINK ARGOBOTS, SWM and UNION
-# target_link_libraries(codes PUBLIC PkgConfig::ARGOBOTS)
-if(USE_ONLINE)
-    if(USE_SWM)
-        target_include_directories(codes PUBLIC ${ARGOBOTS_INCLUDE_DIRS})
-        # target_link_libraries(codes PUBLIC PkgConfig::SWM)
-        target_include_directories(codes PUBLIC ${SWM_INCLUDE_DIRS})
+    if(NOT ZeroMQ_LIBRARY)
+        message(FATAL_ERROR "USE_ZMQML is ON, but libzmq was not found.")
     endif()
-    if(USE_UNION)
-        target_include_directories(codes PUBLIC ${ARGOBOTS_INCLUDE_DIRS})
-        # target_link_libraries(codes PUBLIC PkgConfig::SWM)
-        target_include_directories(codes PUBLIC ${SWM_INCLUDE_DIRS})
-        target_include_directories(codes PUBLIC ${UNION_INCLUDE_DIRS})
-    endif()
-endif()
 
-if(USE_ZMQML)
     add_library(zmqmlrequester SHARED IMPORTED GLOBAL)
     set_target_properties(zmqmlrequester PROPERTIES
         IMPORTED_LOCATION "${ZMQML_BUILD_PATH}/libzmqmlrequester.so"
         INTERFACE_INCLUDE_DIRECTORIES "${ZMQML_BUILD_PATH}")
-    target_compile_definitions(codes PUBLIC USE_ZMQML)
 endif()
 
 #LINK ROSS
 # target_link_libraries(codes PUBLIC #{pkgcfg_lib_ROSS_ROSS})
 # target_link_libraries(codes PUBLIC PkgConfig::ROSS)
+
+add_library(codes ${SRCS})
+
 target_include_directories(codes PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}
     ${ROSS_INCLUDE_DIRS}
@@ -194,11 +165,16 @@ target_include_directories(codes PUBLIC
     ${PROJECT_SOURCE_DIR}/codes
     ${PROJECT_SOURCE_DIR}/src
     ${PROJECT_SOURCE_DIR}/src/modelconfig
-    $<$<BOOL:${USE_ZMQML}>:$<TARGET_PROPERTY:zmqmlrequester,INTERFACE_INCLUDE_DIRECTORIES>>
 )
 
 target_link_libraries(codes PUBLIC ${LIBS_TO_LINK})
 
+if(USE_ZMQML)
+    target_compile_definitions(codes PUBLIC USE_ZMQML)
+    target_include_directories(codes PUBLIC "${ZMQML_BUILD_PATH}" ${ZeroMQ_INCLUDE_DIR})
+    target_link_libraries(codes PUBLIC zmqmlrequester ${ZeroMQ_LIBRARY})
+endif()
+
 get_target_property(CODES_INCLUDE_DIRS codes INCLUDE_DIRECTORIES)
 cmake_print_variables(CODES_INCLUDE_DIRS)
 
@@ -227,18 +203,12 @@ if(USE_DUMPI)
     list(APPEND CODES_TARGETS model-net-dumpi-traces-dump)
 endif()
 
-# ZMQ — only resolved + linked when USE_ZMQML is on; otherwise nothing
-# in the codes library calls into libzmq.
-if(USE_ZMQML)
-    pkg_check_modules(PC_ZeroMQ QUIET zmq)
-    find_path(ZeroMQ_INCLUDE_DIR NAMES zmq.hpp PATHS ${PC_ZeroMQ_INCLUDE_DIRS})
-    find_library(ZeroMQ_LIBRARY NAMES zmq PATHS ${PC_ZeroMQ_LIBRARY_DIRS})
-endif()
-
 foreach(tar IN LISTS CODES_TARGETS)
     target_include_directories(${tar} PUBLIC ${CODES_INCLUDE_DIRS} ${ROSS_INCLUDE_DIRS})
     target_link_libraries(${tar} PUBLIC codes ${LIBS_TO_LINK})
+
     if(USE_ZMQML)
+        target_include_directories(${tar} PUBLIC "${ZMQML_BUILD_PATH}" ${ZeroMQ_INCLUDE_DIR})
         target_link_libraries(${tar} PUBLIC zmqmlrequester ${ZeroMQ_LIBRARY})
     endif()
 endforeach()