00001
00002
00003
00004
00005
00006
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "rmsummary.h"
00023
00024 #define WORK_QUEUE_DEFAULT_PORT 9123
00025 #define WORK_QUEUE_RANDOM_PORT 0
00027 #define WORK_QUEUE_WAITFORTASK -1
00029 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300
00030 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30
00032 typedef enum {
00033 WORK_QUEUE_INPUT = 0,
00034 WORK_QUEUE_OUTPUT = 1
00035 } work_queue_file_type_t;
00036
00037 typedef enum {
00038 WORK_QUEUE_NOCACHE = 0,
00039 WORK_QUEUE_CACHE = 1,
00040 WORK_QUEUE_SYMLINK = 2,
00041 WORK_QUEUE_PREEXIST = 4,
00042 WORK_QUEUE_THIRDGET = 8,
00043 WORK_QUEUE_THIRDPUT = 8,
00044 WORK_QUEUE_WATCH = 16
00045 } work_queue_file_flags_t;
00046
00047 typedef enum {
00048 WORK_QUEUE_SCHEDULE_UNSET = 0,
00049 WORK_QUEUE_SCHEDULE_FCFS,
00050 WORK_QUEUE_SCHEDULE_FILES,
00051 WORK_QUEUE_SCHEDULE_TIME,
00052 WORK_QUEUE_SCHEDULE_RAND,
00053 WORK_QUEUE_SCHEDULE_WORST
00054 } work_queue_schedule_t;
00055
00056
00057 typedef enum {
00058 WORK_QUEUE_RESULT_SUCCESS = 0,
00059 WORK_QUEUE_RESULT_INPUT_MISSING = 1,
00060 WORK_QUEUE_RESULT_OUTPUT_MISSING = 2,
00061 WORK_QUEUE_RESULT_STDOUT_MISSING = 4,
00062 WORK_QUEUE_RESULT_SIGNAL = 8,
00063 WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION = 16,
00064 WORK_QUEUE_RESULT_TASK_TIMEOUT = 32
00065 } work_queue_result_t;
00066
00067 typedef enum {
00068 WORK_QUEUE_TASK_UNKNOWN = 0,
00069 WORK_QUEUE_TASK_READY,
00070 WORK_QUEUE_TASK_RUNNING,
00071 WORK_QUEUE_TASK_WAITING_RETRIEVAL,
00072 WORK_QUEUE_TASK_RETRIEVED,
00073 WORK_QUEUE_TASK_DONE,
00074 WORK_QUEUE_TASK_CANCELED,
00075 WORK_QUEUE_TASK_WAITING_RESUBMISSION
00076 } work_queue_task_state_t;
00077
00078 extern double wq_option_fast_abort_multiplier;
00084 extern int wq_option_scheduler;
00094 struct work_queue_task {
00095 char *tag;
00096 char *command_line;
00097 work_queue_schedule_t worker_selection_algorithm;
00098 char *output;
00099 struct list *input_files;
00100 struct list *output_files;
00101 struct list *env_list;
00102 int taskid;
00103 int return_status;
00104 work_queue_result_t result;
00105 char *host;
00106 char *hostname;
00108 timestamp_t time_committed;
00110 timestamp_t time_task_submit;
00111 timestamp_t time_task_finish;
00112 timestamp_t time_send_input_start;
00113 timestamp_t time_send_input_finish;
00114 timestamp_t time_execute_cmd_start;
00115 timestamp_t time_execute_cmd_finish;
00116 timestamp_t time_receive_result_start;
00117 timestamp_t time_receive_result_finish;
00118 timestamp_t time_receive_output_start;
00119 timestamp_t time_receive_output_finish;
00121 int64_t total_bytes_received;
00122 int64_t total_bytes_sent;
00123 int64_t total_bytes_transferred;
00124 timestamp_t total_transfer_time;
00125 timestamp_t cmd_execution_time;
00126 int total_submissions;
00127 timestamp_t total_cmd_execution_time;
00129 int64_t maximum_end_time;
00130 int64_t memory;
00131 int64_t disk;
00132 int cores;
00133 int gpus;
00134 int unlabeled;
00136 double priority;
00138 struct rmsummary *resources_measured;
00140 timestamp_t time_app_delay;
00142 };
00143
00146 struct work_queue_stats {
00147 int total_workers_connected;
00148 int workers_init;
00149 int workers_idle;
00150 int workers_busy;
00151 int total_workers_joined;
00152 int total_workers_removed;
00154 int tasks_waiting;
00155 int tasks_running;
00156 int tasks_complete;
00157 int total_tasks_dispatched;
00158 int total_tasks_complete;
00159 int total_tasks_failed;
00160 int total_tasks_cancelled;
00162 timestamp_t start_time;
00163 timestamp_t total_send_time;
00164 timestamp_t total_receive_time;
00165 timestamp_t total_good_transfer_time;
00167 timestamp_t total_execute_time;
00168 timestamp_t total_good_execute_time;
00171 int64_t total_bytes_sent;
00172 int64_t total_bytes_received;
00173 double efficiency;
00174 double idle_percentage;
00175 int capacity;
00177 double bandwidth;
00178 int64_t total_cores;
00179 int64_t total_memory;
00180 int64_t total_disk;
00181 int64_t total_gpus;
00182 int64_t committed_cores;
00183 int64_t committed_memory;
00184 int64_t committed_disk;
00185 int64_t committed_gpus;
00186 int64_t min_cores;
00187 int64_t max_cores;
00188 int64_t min_memory;
00189 int64_t max_memory;
00190 int64_t min_disk;
00191 int64_t max_disk;
00192 int64_t min_gpus;
00193 int64_t max_gpus;
00194 int port;
00195 int priority;
00196 int workers_ready;
00197 int workers_full;
00198 int total_worker_slots;
00199 int avg_capacity;
00200 };
00201
00202
00206
00214 struct work_queue_task *work_queue_task_create(const char *full_command);
00215
00221 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00222
00227 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00228
00245 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t flags);
00246
00261 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, work_queue_file_type_t type, work_queue_file_flags_t flags);
00262
00273 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, work_queue_file_flags_t);
00274
00288 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t, int recursive);
00289
00295 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00296
00302 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00303
00309 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00310
00316 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00317
00323 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00324
00331 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00332
00339 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00340
00347 void work_queue_task_specify_enviroment_variable( struct work_queue_task *t, const char *name, const char *value );
00348
00354 void work_queue_task_specify_algorithm(struct work_queue_task *t, work_queue_schedule_t algorithm);
00355
00360 void work_queue_task_delete(struct work_queue_task *t);
00361
00363
00367
00384 struct work_queue *work_queue_create(int port);
00385
00393 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00394
00403 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00404
00409 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00410
00411
00416 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00417
00418
00422 void work_queue_blacklist_clear(struct work_queue *q);
00423
00438 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00439
00451 int work_queue_hungry(struct work_queue *q);
00452
00460 int work_queue_empty(struct work_queue *q);
00461
00468 int work_queue_port(struct work_queue *q);
00469
00474 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00475
00480 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00481
00482
00488 work_queue_task_state_t work_queue_task_state(struct work_queue *q, int taskid);
00489
00494 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00495
00500 double work_queue_get_effective_bandwidth(struct work_queue *q);
00501
00508 char * work_queue_get_worker_summary( struct work_queue *q );
00509
00515 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00516
00517
00521 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00522
00528 void work_queue_specify_algorithm(struct work_queue *q, work_queue_schedule_t algorithm);
00529
00534 const char *work_queue_name(struct work_queue *q);
00535
00540 void work_queue_specify_name(struct work_queue *q, const char *name);
00541
00546 void work_queue_specify_priority(struct work_queue *q, int priority);
00547
00556 void work_queue_specify_num_tasks_left(struct work_queue *q, int ntasks);
00557
00563 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00564
00570 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00571
00577 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00578
00583 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00584
00589 int work_queue_shut_down_workers(struct work_queue *q, int n);
00590
00595 void work_queue_delete(struct work_queue *q);
00596
00602 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00603
00609 void work_queue_specify_password( struct work_queue *q, const char *password );
00610
00617 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00618
00623 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00624
00629 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00630
00636 void work_queue_master_preferred_connection(struct work_queue *q, const char *preferred_connection);
00637
00653 int work_queue_tune(struct work_queue *q, const char *name, double value);
00654
00656
00660
00661 #define WORK_QUEUE_TASK_ORDER_FIFO 0
00662 #define WORK_QUEUE_TASK_ORDER_LIFO 1
00670 void work_queue_specify_task_order(struct work_queue *q, int order);
00671
00672
00673 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0
00674 #define WORK_QUEUE_MASTER_MODE_CATALOG 1
00683 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00684
00685
00691 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00692
00701 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00702
00710 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00711
00719 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00720
00728 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00729
00737 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00738
00740
00741
00742
00743
00744
00745
00746
00747
00748 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00749
00750 #endif
00751
00752
00753