FM / runtime

/usr/lib/x86_64-linux-gnu/openmpi/include/openmpi/opal/runtime/

/usr/lib/x86_64-linux-gnu/openmpi/include/openmpi/opal/runtime

Name	Size	Mode	Actions
opal.h	4046	0644	edit dl rm
opal_cr.h	14888	0644	edit dl rm
opal_info_support.h	4431	0644	edit dl rm
opal_params.h	3029	0644	edit dl rm
opal_progress.h	6398	0644	edit dl rm
opal_progress_threads.h	2243	0644	edit dl rm

Edit: /usr/lib/x86_64-linux-gnu/openmpi/include/openmpi/opal/runtime/opal_cr.h (14888B)

/*
 * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2005 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
 * Copyright (c) 2017      IBM Corporation. All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

/**
 * @file
 *
 * Checkpoint functionality for Open MPI
 */

#include "opal_config.h"
#include "opal/mca/crs/crs.h"
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/prefetch.h"

#ifndef OPAL_CR_H
#define OPAL_CR_H

BEGIN_C_DECLS

/*
 * Some defines shared with opal-[checkpoint|restart] commands
 */
#define OPAL_CR_DONE       ((char) 0)
#define OPAL_CR_ACK        ((char) 1)
#define OPAL_CR_CHECKPOINT ((char) 2)
#define OPAL_CR_NAMED_PROG_R  ("opal_cr_prog_read")
#define OPAL_CR_NAMED_PROG_W  ("opal_cr_prog_write")
#define OPAL_CR_BASE_ENV_NAME ("opal_cr_restart-env")

/*
 * Possible responses to a checkpoint request from opal-checkpoint
 */
enum opal_cr_ckpt_cmd_state_t {
    OPAL_CHECKPOINT_CMD_START,       /* Checkpoint is starting on this request */
    OPAL_CHECKPOINT_CMD_IN_PROGRESS, /* Checkpoint is currently running */
    OPAL_CHECKPOINT_CMD_NULL,        /* Checkpoint cannot be started because it is not supported */
    OPAL_CHECKPOINT_CMD_ERROR,       /* An error occurred such that the checkpoint cannot be completed */
    /* State of the checkpoint operation */
    OPAL_CR_STATUS_NONE,       /* No checkpoint in progress */
    OPAL_CR_STATUS_REQUESTED,  /* Checkpoint has been requested */
    OPAL_CR_STATUS_RUNNING,    /* Checkpoint is currently running */
    OPAL_CR_STATUS_TERM,       /* Checkpoint is running and will terminate process upon completion */
    /* State of the continue operation */
    OPAL_CR_STATUS_CONTINUE,
    /* State of the restart operation */
    OPAL_CR_STATUS_RESTART_PRE,
    OPAL_CR_STATUS_RESTART_POST
};
typedef enum opal_cr_ckpt_cmd_state_t opal_cr_ckpt_cmd_state_t;

/* An output handle to be used by the cr runtime
     * functionality as an argument to opal_output() */
    OPAL_DECLSPEC extern int    opal_cr_output;

/* Directory containing the named pipes for communication
     * with the opal-checkpoint tool  */
    OPAL_DECLSPEC extern char * opal_cr_pipe_dir;

/* Signal that opal-checkpoint uses to contact the
     * application process */
    OPAL_DECLSPEC extern int    opal_cr_entry_point_signal;

/* If Checkpointing is enabled in this application */
    OPAL_DECLSPEC extern bool   opal_cr_is_enabled;

/* If the application running is a tool
     * (e.g., opal-checkpoint, orted, ...) */
    OPAL_DECLSPEC extern bool   opal_cr_is_tool;

/* If a checkpoint has been requested */
    OPAL_DECLSPEC extern int opal_cr_checkpoint_request;

/* The current state of a checkpoint operation */
    OPAL_DECLSPEC extern int opal_cr_checkpointing_state;

/*
     * If one of the BTLs that shutdown require a full, clean rebuild of the
     * point-to-point stack on 'continue' as well as 'restart'.
     */
    OPAL_DECLSPEC extern bool opal_cr_continue_like_restart;

#if OPAL_ENABLE_CRDEBUG == 1
    /* Whether or not C/R Debugging is enabled for this process */
    OPAL_DECLSPEC extern int MPIR_debug_with_checkpoint;

/*
     * Set/clear the current thread id for the checkpointing thread
     */
    OPAL_DECLSPEC int opal_cr_debug_set_current_ckpt_thread_self(void);
    OPAL_DECLSPEC int opal_cr_debug_clear_current_ckpt_thread(void);

/*
     * This MPI Debugger function needs to be accessed here and have a specific
     * name. Thus we are breaking the traditional naming conventions to provide this functionality.
     */
    OPAL_DECLSPEC int MPIR_checkpoint_debugger_detach(void);

/**
     * A tight loop to wait for debugger to release this process from the
     * breakpoint.
     */
    OPAL_DECLSPEC void *MPIR_checkpoint_debugger_breakpoint(void);

/**
     * A function for the debugger or CRS to force all threads into
     */
    OPAL_DECLSPEC void *MPIR_checkpoint_debugger_waitpoint(void);

/**
     * A signal handler to force all threads to wait when debugger detaches
     */
    OPAL_DECLSPEC void MPIR_checkpoint_debugger_signal_handler(int signo);
#endif

/*
     * Refresh environment variables after a restart
     */
    OPAL_DECLSPEC int opal_cr_refresh_environ(int prev_pid);

/*
     * If this is an application that doesn't want to have
     * a notification callback installed, set this to false.
     * To see the effect, this must be called before opal_cr_init().
     * Default: Enabled
     */
    OPAL_DECLSPEC int opal_cr_set_enabled(bool);

/**
     * Initialize the notification and coordination
     *  elements.
     */
    OPAL_DECLSPEC int opal_cr_init(void);

/**
     * Finalize the notification and coordination
     *  elements.
     */
    OPAL_DECLSPEC int opal_cr_finalize(void);

/*************************************************
     * Check to see if a checkpoint has been requested
     *
     * When the checkpoint thread is disabled:
     *   This will be checked whenever the MPI Library
     *   is entered by the application. It will stop
     *   the application for the duration of the entire
     *   checkpoint.
     * When the checkpoint thread is enabled:
     *   The request is handled in the thread parallel
     *   with the execution of the program regardless
     *   of where the program is in exection.
     *   The problem with this method is that it
     *   requires the support of progress threads
     *   which is currently not working properly :/
     *
     *************************************************/
    OPAL_DECLSPEC void opal_cr_test_if_checkpoint_ready(void);

/* If the checkpoint operation should be stalled to
     * wait for another sevice to complete before
     * continuing with the checkpoint */
    OPAL_DECLSPEC extern bool opal_cr_stall_check;
    OPAL_DECLSPEC extern bool opal_cr_currently_stalled;

#if OPAL_ENABLE_FT_THREAD == 1
    /* Some thread functions */
    OPAL_DECLSPEC void opal_cr_thread_init_library(void);
    OPAL_DECLSPEC void opal_cr_thread_finalize_library(void);
    OPAL_DECLSPEC void opal_cr_thread_abort_library(void);
    OPAL_DECLSPEC void opal_cr_thread_enter_library(void);
    OPAL_DECLSPEC void opal_cr_thread_exit_library(void);
    OPAL_DECLSPEC void opal_cr_thread_noop_progress(void);
#endif /* OPAL_ENABLE_FT_THREAD == 1 */

/*
     * If not using FT then make the #defines noops
     */
#if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0
#define OPAL_CR_TEST_CHECKPOINT_READY() ;
#define OPAL_CR_TEST_CHECKPOINT_READY_STALL() ;
#define OPAL_CR_INIT_LIBRARY() ;
#define OPAL_CR_FINALIZE_LIBRARY() ;
#define OPAL_CR_ABORT_LIBRARY() ;
#define OPAL_CR_ENTER_LIBRARY() ;
#define OPAL_CR_EXIT_LIBRARY() ;
#define OPAL_CR_NOOP_PROGRESS() ;
#endif /* #if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0 */

/*
     * If using FT
     */
#if OPAL_ENABLE_FT_CR == 1
#define OPAL_CR_TEST_CHECKPOINT_READY()      \
  {                                          \
    if(OPAL_UNLIKELY(opal_cr_is_enabled) ) { \
      opal_cr_test_if_checkpoint_ready();    \
    }                                        \
  }

#define OPAL_CR_TEST_CHECKPOINT_READY_STALL()        \
  {                                                  \
    if(OPAL_UNLIKELY(opal_cr_is_enabled && !opal_cr_stall_check)) { \
      opal_cr_test_if_checkpoint_ready();            \
    }                                                \
  }

/* If *not* using FT thread */
#if OPAL_ENABLE_FT_THREAD == 0
#define OPAL_CR_INIT_LIBRARY()     OPAL_CR_TEST_CHECKPOINT_READY();
#define OPAL_CR_FINALIZE_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
#define OPAL_CR_ABORT_LIBRARY()    OPAL_CR_TEST_CHECKPOINT_READY();
#define OPAL_CR_ENTER_LIBRARY()    OPAL_CR_TEST_CHECKPOINT_READY();
#define OPAL_CR_EXIT_LIBRARY()     OPAL_CR_TEST_CHECKPOINT_READY();
#define OPAL_CR_NOOP_PROGRESS()    OPAL_CR_TEST_CHECKPOINT_READY();
#endif /* OPAL_ENABLE_FT_THREAD == 0 */

/* If using FT thread */
#if OPAL_ENABLE_FT_THREAD == 1
#define OPAL_CR_INIT_LIBRARY()    \
 {                                \
   opal_cr_thread_init_library(); \
 }
#define OPAL_CR_FINALIZE_LIBRARY()    \
 {                                    \
   opal_cr_thread_finalize_library(); \
 }
#define OPAL_CR_ABORT_LIBRARY()    \
 {                                 \
   opal_cr_thread_abort_library(); \
 }
#define OPAL_CR_ENTER_LIBRARY()    \
 {                                 \
   opal_cr_thread_enter_library(); \
 }
#define OPAL_CR_EXIT_LIBRARY()    \
 {                                \
   opal_cr_thread_exit_library(); \
 }
#define OPAL_CR_NOOP_PROGRESS()    \
 {                                 \
   opal_cr_thread_noop_progress(); \
 }
#endif /* OPAL_ENABLE_FT_THREAD == 1 */

#endif /* OPAL_ENABLE_FT_CR == 1 */

/*******************************
     * Notification Routines
     *******************************/
    /*******************************
     * Notification Routines
     *******************************/
    /**
     * A function to respond to the async checkpoint request
     * this is useful when figuring out who should respond
     * when stalling.
     */
    typedef int (*opal_cr_notify_callback_fn_t) (opal_cr_ckpt_cmd_state_t);

OPAL_DECLSPEC int opal_cr_reg_notify_callback
    (opal_cr_notify_callback_fn_t new_func,
     opal_cr_notify_callback_fn_t *prev_func);

/**
     * Function to go through the INC
     * - Call Registered INC_Coord(CHECKPOINT)
     * - Call the CRS.checkpoint()
     * - Call Registered INC_Coord(state)
     */
    OPAL_DECLSPEC int opal_cr_inc_core(pid_t pid,
                                       opal_crs_base_snapshot_t *snapshot,
                                       opal_crs_base_ckpt_options_t *options,
                                       int *state);

OPAL_DECLSPEC int opal_cr_inc_core_prep(void);
    OPAL_DECLSPEC int opal_cr_inc_core_ckpt(pid_t pid,
                                            opal_crs_base_snapshot_t *snapshot,
                                            opal_crs_base_ckpt_options_t *options,
                                            int *state);
    OPAL_DECLSPEC int opal_cr_inc_core_recover(int state);

/*******************************
     * User Coordination Routines
     *******************************/
    typedef enum {
        OPAL_CR_INC_PRE_CRS_PRE_MPI   = 0,
        OPAL_CR_INC_PRE_CRS_POST_MPI  = 1,
        OPAL_CR_INC_CRS_PRE_CKPT      = 2,
        OPAL_CR_INC_CRS_POST_CKPT     = 3,
        OPAL_CR_INC_POST_CRS_PRE_MPI  = 4,
        OPAL_CR_INC_POST_CRS_POST_MPI = 5,
        OPAL_CR_INC_MAX               = 6
    } opal_cr_user_inc_callback_event_t;

typedef enum {
        OPAL_CR_INC_STATE_PREPARE  = 0,
        OPAL_CR_INC_STATE_CONTINUE = 1,
        OPAL_CR_INC_STATE_RESTART  = 2,
        OPAL_CR_INC_STATE_ERROR    = 3
    } opal_cr_user_inc_callback_state_t;

/**
     * User coordination callback routine
     */
    typedef int (*opal_cr_user_inc_callback_fn_t)(opal_cr_user_inc_callback_event_t event,
                                                  opal_cr_user_inc_callback_state_t state);

OPAL_DECLSPEC int opal_cr_user_inc_register_callback
                      (opal_cr_user_inc_callback_event_t event,
                       opal_cr_user_inc_callback_fn_t  function,
                       opal_cr_user_inc_callback_fn_t  *prev_function);

OPAL_DECLSPEC int ompi_trigger_user_inc_callback(opal_cr_user_inc_callback_event_t event,
                                                opal_cr_user_inc_callback_state_t state);

/*******************************
     * Coordination Routines
     *******************************/
    /**
     * Coordination callback routine signature
     */
    typedef int (*opal_cr_coord_callback_fn_t) (int);

/**
     * Register a checkpoint coodination routine
     * for a higher level.
     */
     OPAL_DECLSPEC int opal_cr_reg_coord_callback
     (opal_cr_coord_callback_fn_t  new_func,
      opal_cr_coord_callback_fn_t *prev_func);

/**
     * OPAL Checkpoint Coordination Routine
     */
    OPAL_DECLSPEC int opal_cr_coord(int state);

/**
     * Checkpoint life-cycle timing
     */
    OPAL_DECLSPEC void opal_cr_set_time(int idx);
    OPAL_DECLSPEC void opal_cr_display_all_timers(void);
    OPAL_DECLSPEC void opal_cr_clear_timers(void);

OPAL_DECLSPEC extern bool opal_cr_timing_enabled;
    OPAL_DECLSPEC extern bool opal_cr_timing_barrier_enabled;
    OPAL_DECLSPEC extern int  opal_cr_timing_my_rank;
    OPAL_DECLSPEC extern int  opal_cr_timing_target_rank;

#define OPAL_CR_TIMER_ENTRY0    0
#define OPAL_CR_TIMER_ENTRY1    1
#define OPAL_CR_TIMER_ENTRY2    2
#define OPAL_CR_TIMER_CRCPBR0   3
#define OPAL_CR_TIMER_CRCP0     4
#define OPAL_CR_TIMER_CRCPBR1   5
#define OPAL_CR_TIMER_P2P0      6
#define OPAL_CR_TIMER_P2P1      7
#define OPAL_CR_TIMER_P2PBR0    8
#define OPAL_CR_TIMER_CORE0     9
#define OPAL_CR_TIMER_CORE1    10
#define OPAL_CR_TIMER_COREBR0  11
#define OPAL_CR_TIMER_P2P2     12
#define OPAL_CR_TIMER_P2PBR1   13
#define OPAL_CR_TIMER_P2P3     14
#define OPAL_CR_TIMER_P2PBR2   15
#define OPAL_CR_TIMER_CRCP1    16
#define OPAL_CR_TIMER_COREBR1  17
#define OPAL_CR_TIMER_CORE2    18
#define OPAL_CR_TIMER_ENTRY3   19
#define OPAL_CR_TIMER_ENTRY4   20
#define OPAL_CR_TIMER_MAX      21

#define OPAL_CR_CLEAR_TIMERS()                          \
    {                                                   \
        if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
            opal_cr_clear_timers();                     \
        }                                               \
    }

#define OPAL_CR_SET_TIMER(idx)                          \
    {                                                   \
        if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
            opal_cr_set_time(idx);                      \
        }                                               \
    }

#define OPAL_CR_DISPLAY_ALL_TIMERS()                    \
    {                                                   \
        if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
            opal_cr_display_all_timers();               \
        }                                               \
    }

END_C_DECLS

#endif /* OPAL_CR_H */

cmd: