00001
00002
00003
00004
00005
00006
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022
00023 #define WORK_QUEUE_DEFAULT_PORT 9123
00024 #define WORK_QUEUE_RANDOM_PORT 0
00025 #define WORK_QUEUE_WAITFORTASK -1
00027 #define WORK_QUEUE_SCHEDULE_UNSET 0
00028 #define WORK_QUEUE_SCHEDULE_FCFS 1
00029 #define WORK_QUEUE_SCHEDULE_FILES 2
00030 #define WORK_QUEUE_SCHEDULE_TIME 3
00031 #define WORK_QUEUE_SCHEDULE_RAND 4
00033 #define WORK_QUEUE_TASK_ORDER_FIFO 0
00034 #define WORK_QUEUE_TASK_ORDER_LIFO 1
00036 #define WORK_QUEUE_INPUT 0
00037 #define WORK_QUEUE_OUTPUT 1
00039 #define WORK_QUEUE_NOCACHE 0
00040 #define WORK_QUEUE_CACHE 1
00041 #define WORK_QUEUE_SYMLINK 2
00042 #define WORK_QUEUE_PREEXIST 4
00043 #define WORK_QUEUE_THIRDGET 8
00044 #define WORK_QUEUE_THIRDPUT 8
00045
00046 #define WORK_QUEUE_RESET_ALL 0
00047 #define WORK_QUEUE_RESET_KEEP_TASKS 1
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300
00050 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30
00052 #define WORK_QUEUE_RESULT_SUCCESS 0
00053 #define WORK_QUEUE_RESULT_INPUT_MISSING 1
00054 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2
00055 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4
00057 extern double wq_option_fast_abort_multiplier;
00059 extern int wq_option_scheduler;
00063 struct work_queue_task {
00064 char *tag;
00065 char *command_line;
00066 int worker_selection_algorithm;
00067 char *output;
00068 struct list *input_files;
00069 struct list *output_files;
00070 int taskid;
00071 int return_status;
00072 int result;
00073 char *host;
00074 char *hostname;
00076 timestamp_t time_committed;
00078 timestamp_t time_task_submit;
00079 timestamp_t time_task_finish;
00080 timestamp_t time_send_input_start;
00081 timestamp_t time_send_input_finish;
00082 timestamp_t time_execute_cmd_start;
00083 timestamp_t time_execute_cmd_finish;
00084 timestamp_t time_receive_result_start;
00085 timestamp_t time_receive_result_finish;
00086 timestamp_t time_receive_output_start;
00087 timestamp_t time_receive_output_finish;
00089 int64_t total_bytes_received;
00090 int64_t total_bytes_sent;
00091 int64_t total_bytes_transferred;
00092 timestamp_t total_transfer_time;
00093 timestamp_t cmd_execution_time;
00094 int total_submissions;
00095 timestamp_t total_cmd_execution_time;
00097 int64_t memory;
00098 int64_t disk;
00099 int cores;
00100 int gpus;
00101 int unlabeled;
00102
00103 timestamp_t time_app_delay;
00104 };
00105
00108 struct work_queue_stats {
00109 int total_workers_connected;
00110 int workers_init;
00111 int workers_idle;
00112 int workers_busy;
00113 int total_workers_joined;
00114 int total_workers_removed;
00116 int tasks_waiting;
00117 int tasks_running;
00118 int tasks_complete;
00119 int total_tasks_dispatched;
00120 int total_tasks_complete;
00121 int total_tasks_cancelled;
00123 timestamp_t start_time;
00124 timestamp_t total_send_time;
00125 timestamp_t total_receive_time;
00126 int64_t total_bytes_sent;
00127 int64_t total_bytes_received;
00128 double efficiency;
00129 double idle_percentage;
00130 int capacity;
00132 double bandwidth;
00133 int64_t total_cores;
00134 int64_t total_memory;
00135 int64_t total_disk;
00136 int64_t total_gpus;
00137 int64_t committed_cores;
00138 int64_t committed_memory;
00139 int64_t committed_disk;
00140 int64_t committed_gpus;
00141 int64_t min_cores;
00142 int64_t max_cores;
00143 int64_t min_memory;
00144 int64_t max_memory;
00145 int64_t min_disk;
00146 int64_t max_disk;
00147 int64_t min_gpus;
00148 int64_t max_gpus;
00149 timestamp_t total_execute_time;
00150 timestamp_t total_good_execute_time;
00152 int port;
00153 int priority;
00154 int workers_ready;
00155 int workers_full;
00156 int total_worker_slots;
00157 int avg_capacity;
00158 };
00159
00160
00164
00172 struct work_queue_task *work_queue_task_create(const char *full_command);
00173
00179 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00180
00185 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00186
00199 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00200
00215 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00216
00227 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00228
00242 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00243
00249 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00250
00256 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00257
00263 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00264
00270 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00271
00278 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00279
00289 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00290
00295 void work_queue_task_delete(struct work_queue_task *t);
00296
00298
00302
00319 struct work_queue *work_queue_create(int port);
00320
00328 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00329
00338 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00339
00354 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00355
00367 int work_queue_hungry(struct work_queue *q);
00368
00376 int work_queue_empty(struct work_queue *q);
00377
00384 int work_queue_port(struct work_queue *q);
00385
00390 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00391
00396 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00397
00402 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00403
00408 double work_queue_get_effective_bandwidth(struct work_queue *q);
00409
00416 char * work_queue_get_worker_summary( struct work_queue *q );
00417
00423 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00424
00425
00437 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00438
00448 void work_queue_specify_task_order(struct work_queue *q, int order);
00449
00454 const char *work_queue_name(struct work_queue *q);
00455
00460 void work_queue_specify_name(struct work_queue *q, const char *name);
00461
00466 void work_queue_specify_priority(struct work_queue *q, int priority);
00467
00473 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00474
00480 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00481
00487 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00488
00493 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00494
00501 void work_queue_reset(struct work_queue *q, int flags);
00502
00507 int work_queue_shut_down_workers(struct work_queue *q, int n);
00508
00513 void work_queue_delete(struct work_queue *q);
00514
00520 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00521
00527 void work_queue_specify_password( struct work_queue *q, const char *password );
00528
00535 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00536
00541 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00542
00547 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00548
00549
00565 int work_queue_tune(struct work_queue *q, const char *name, double value);
00566
00568
00572
00573 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0
00574 #define WORK_QUEUE_MASTER_MODE_CATALOG 1
00583 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00584
00590 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00591
00600 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00601
00609 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00610
00618 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00619
00627 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00628
00636 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00637
00639
00640
00641
00642
00643
00644
00645
00646
00647 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00648
00649 #endif