00001
00002
00003
00004
00005
00006
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "rmsummary.h"
00023
00024 #define WORK_QUEUE_DEFAULT_PORT 9123
00025 #define WORK_QUEUE_RANDOM_PORT 0
00026 #define WORK_QUEUE_WAITFORTASK -1
00028 #define WORK_QUEUE_SCHEDULE_UNSET 0
00029 #define WORK_QUEUE_SCHEDULE_FCFS 1
00030 #define WORK_QUEUE_SCHEDULE_FILES 2
00031 #define WORK_QUEUE_SCHEDULE_TIME 3
00032 #define WORK_QUEUE_SCHEDULE_RAND 4
00033 #define WORK_QUEUE_SCHEDULE_WORST 5
00035 #define WORK_QUEUE_INPUT 0
00036 #define WORK_QUEUE_OUTPUT 1
00038 #define WORK_QUEUE_NOCACHE 0
00039 #define WORK_QUEUE_CACHE 1
00040 #define WORK_QUEUE_SYMLINK 2
00041 #define WORK_QUEUE_PREEXIST 4
00042 #define WORK_QUEUE_THIRDGET 8
00043 #define WORK_QUEUE_THIRDPUT 8
00044 #define WORK_QUEUE_WATCH 16
00046 #define WORK_QUEUE_RESET_ALL 0
00047 #define WORK_QUEUE_RESET_KEEP_TASKS 1
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300
00050 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30
00052 #define WORK_QUEUE_RESULT_SUCCESS 0
00053 #define WORK_QUEUE_RESULT_INPUT_MISSING 1
00054 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2
00055 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4
00056 #define WORK_QUEUE_RESULT_SIGNAL 8
00057 #define WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16
00058 #define WORK_QUEUE_RESULT_TASK_TIMEOUT 32
00061 #define WORK_QUEUE_TASK_UNKNOWN 0
00062 #define WORK_QUEUE_TASK_READY 1
00063 #define WORK_QUEUE_TASK_RUNNING 2
00064 #define WORK_QUEUE_TASK_WAITING_RETRIEVAL 3
00065 #define WORK_QUEUE_TASK_RETRIEVED 4
00066 #define WORK_QUEUE_TASK_DONE 5
00067 #define WORK_QUEUE_TASK_CANCELED 6
00069 extern double wq_option_fast_abort_multiplier;
00071 extern int wq_option_scheduler;
00075 struct work_queue_task {
00076 char *tag;
00077 char *command_line;
00078 int worker_selection_algorithm;
00079 char *output;
00080 struct list *input_files;
00081 struct list *output_files;
00082 struct list *env_list;
00083 int taskid;
00084 int return_status;
00085 int result;
00086 char *host;
00087 char *hostname;
00089 timestamp_t time_committed;
00091 timestamp_t time_task_submit;
00092 timestamp_t time_task_finish;
00093 timestamp_t time_send_input_start;
00094 timestamp_t time_send_input_finish;
00095 timestamp_t time_execute_cmd_start;
00096 timestamp_t time_execute_cmd_finish;
00097 timestamp_t time_receive_result_start;
00098 timestamp_t time_receive_result_finish;
00099 timestamp_t time_receive_output_start;
00100 timestamp_t time_receive_output_finish;
00102 int64_t total_bytes_received;
00103 int64_t total_bytes_sent;
00104 int64_t total_bytes_transferred;
00105 timestamp_t total_transfer_time;
00106 timestamp_t cmd_execution_time;
00107 int total_submissions;
00108 timestamp_t total_cmd_execution_time;
00110 int64_t maximum_end_time;
00111 int64_t memory;
00112 int64_t disk;
00113 int cores;
00114 int gpus;
00115 int unlabeled;
00116
00117 double priority;
00119 struct rmsummary *resources_measured;
00121 timestamp_t time_app_delay;
00123 };
00124
00127 struct work_queue_stats {
00128 int total_workers_connected;
00129 int workers_init;
00130 int workers_idle;
00131 int workers_busy;
00132 int total_workers_joined;
00133 int total_workers_removed;
00135 int tasks_waiting;
00136 int tasks_running;
00137 int tasks_complete;
00138 int total_tasks_dispatched;
00139 int total_tasks_complete;
00140 int total_tasks_failed;
00141 int total_tasks_cancelled;
00143 timestamp_t start_time;
00144 timestamp_t total_send_time;
00145 timestamp_t total_receive_time;
00146 timestamp_t total_good_transfer_time;
00148 timestamp_t total_execute_time;
00149 timestamp_t total_good_execute_time;
00152 int64_t total_bytes_sent;
00153 int64_t total_bytes_received;
00154 double efficiency;
00155 double idle_percentage;
00156 int capacity;
00158 double bandwidth;
00159 int64_t total_cores;
00160 int64_t total_memory;
00161 int64_t total_disk;
00162 int64_t total_gpus;
00163 int64_t committed_cores;
00164 int64_t committed_memory;
00165 int64_t committed_disk;
00166 int64_t committed_gpus;
00167 int64_t min_cores;
00168 int64_t max_cores;
00169 int64_t min_memory;
00170 int64_t max_memory;
00171 int64_t min_disk;
00172 int64_t max_disk;
00173 int64_t min_gpus;
00174 int64_t max_gpus;
00175 int port;
00176 int priority;
00177 int workers_ready;
00178 int workers_full;
00179 int total_worker_slots;
00180 int avg_capacity;
00181 };
00182
00183
00187
00195 struct work_queue_task *work_queue_task_create(const char *full_command);
00196
00202 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00203
00208 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00209
00226 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00227
00242 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00243
00254 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00255
00269 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00270
00276 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00277
00283 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00284
00290 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00291
00297 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00298
00304 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00305
00312 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00313
00320 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00321
00328 void work_queue_task_specify_env( struct work_queue_task *t, const char *name, const char *value );
00329
00339 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00340
00345 void work_queue_task_delete(struct work_queue_task *t);
00346
00348
00352
00369 struct work_queue *work_queue_create(int port);
00370
00378 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00379
00388 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00389
00394 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00395
00396
00401 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00402
00403
00407 void work_queue_blacklist_clear(struct work_queue *q);
00408
00423 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00424
00436 int work_queue_hungry(struct work_queue *q);
00437
00445 int work_queue_empty(struct work_queue *q);
00446
00453 int work_queue_port(struct work_queue *q);
00454
00459 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00460
00465 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00466
00467
00473 int work_queue_task_state(struct work_queue *q, int taskid);
00474
00479 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00480
00485 double work_queue_get_effective_bandwidth(struct work_queue *q);
00486
00493 char * work_queue_get_worker_summary( struct work_queue *q );
00494
00500 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00501
00502
00506 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00507
00517 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00518
00523 const char *work_queue_name(struct work_queue *q);
00524
00529 void work_queue_specify_name(struct work_queue *q, const char *name);
00530
00535 void work_queue_specify_priority(struct work_queue *q, int priority);
00536
00542 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00543
00549 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00550
00556 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00557
00562 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00563
00568 int work_queue_shut_down_workers(struct work_queue *q, int n);
00569
00574 void work_queue_delete(struct work_queue *q);
00575
00581 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00582
00588 void work_queue_specify_password( struct work_queue *q, const char *password );
00589
00596 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00597
00602 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00603
00608 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00609
00610
00626 int work_queue_tune(struct work_queue *q, const char *name, double value);
00627
00629
00633
00634 #define WORK_QUEUE_TASK_ORDER_FIFO 0
00635 #define WORK_QUEUE_TASK_ORDER_LIFO 1
00643 void work_queue_specify_task_order(struct work_queue *q, int order);
00644
00645
00646 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0
00647 #define WORK_QUEUE_MASTER_MODE_CATALOG 1
00656 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00657
00663 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00664
00673 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00674
00682 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00683
00691 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00692
00700 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00701
00709 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00710
00712
00713
00714
00715
00716
00717
00718
00719
00720 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00721
00722 #endif