00001
00002
00003
00004
00005
00006
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "category.h"
00023 #include "rmsummary.h"
00024
00025 #define WORK_QUEUE_DEFAULT_PORT 9123
00026 #define WORK_QUEUE_RANDOM_PORT 0
00028 #define WORK_QUEUE_WAITFORTASK -1
00030 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 120
00031 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30
00033 typedef enum {
00034 WORK_QUEUE_INPUT = 0,
00035 WORK_QUEUE_OUTPUT = 1
00036 } work_queue_file_type_t;
00037
00038 typedef enum {
00039 WORK_QUEUE_NOCACHE = 0,
00040 WORK_QUEUE_CACHE = 1,
00041 WORK_QUEUE_SYMLINK = 2,
00042 WORK_QUEUE_PREEXIST = 4,
00043 WORK_QUEUE_THIRDGET = 8,
00044 WORK_QUEUE_THIRDPUT = 8,
00045 WORK_QUEUE_WATCH = 16
00046 } work_queue_file_flags_t;
00047
00048 typedef enum {
00049 WORK_QUEUE_SCHEDULE_UNSET = 0,
00050 WORK_QUEUE_SCHEDULE_FCFS,
00051 WORK_QUEUE_SCHEDULE_FILES,
00052 WORK_QUEUE_SCHEDULE_TIME,
00053 WORK_QUEUE_SCHEDULE_RAND,
00054 WORK_QUEUE_SCHEDULE_WORST
00055 } work_queue_schedule_t;
00056
00057
00058 typedef enum {
00059 WORK_QUEUE_RESULT_SUCCESS = 0,
00060 WORK_QUEUE_RESULT_INPUT_MISSING = 1,
00061 WORK_QUEUE_RESULT_OUTPUT_MISSING = 2,
00062 WORK_QUEUE_RESULT_STDOUT_MISSING = 4,
00063 WORK_QUEUE_RESULT_SIGNAL = 8,
00064 WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION = 16,
00065 WORK_QUEUE_RESULT_TASK_TIMEOUT = 32,
00066 WORK_QUEUE_RESULT_UNKNOWN = 64,
00067 WORK_QUEUE_RESULT_FORSAKEN = 128,
00068 WORK_QUEUE_RESULT_MAX_RETRIES = 256,
00069 WORK_QUEUE_RESULT_TASK_MAX_RUN_TIME = 512
00070 } work_queue_result_t;
00071
00072 typedef enum {
00073 WORK_QUEUE_TASK_UNKNOWN = 0,
00074 WORK_QUEUE_TASK_READY,
00075 WORK_QUEUE_TASK_RUNNING,
00076 WORK_QUEUE_TASK_WAITING_RETRIEVAL,
00077 WORK_QUEUE_TASK_RETRIEVED,
00078 WORK_QUEUE_TASK_DONE,
00079 WORK_QUEUE_TASK_CANCELED,
00080 WORK_QUEUE_TASK_WAITING_RESUBMISSION
00081 } work_queue_task_state_t;
00082
00083 typedef enum {
00084 WORK_QUEUE_FILE = 1,
00085 WORK_QUEUE_BUFFER,
00086 WORK_QUEUE_REMOTECMD,
00087 WORK_QUEUE_FILE_PIECE,
00088 WORK_QUEUE_DIRECTORY,
00089 WORK_QUEUE_URL
00090 } work_queue_file_t;
00091
00092 extern int wq_option_scheduler;
00102 struct work_queue_task {
00103 char *tag;
00104 char *command_line;
00105 work_queue_schedule_t worker_selection_algorithm;
00106 char *output;
00107 struct list *input_files;
00108 struct list *output_files;
00109 struct list *env_list;
00110 int taskid;
00111 int return_status;
00112 work_queue_result_t result;
00113 char *host;
00114 char *hostname;
00116 timestamp_t time_committed;
00118 timestamp_t time_task_submit;
00119 timestamp_t time_task_finish;
00120 timestamp_t time_send_input_start;
00121 timestamp_t time_send_input_finish;
00122 timestamp_t time_execute_cmd_start;
00123 timestamp_t time_execute_cmd_finish;
00124 timestamp_t time_receive_result_start;
00125 timestamp_t time_receive_result_finish;
00126 timestamp_t time_receive_output_start;
00127 timestamp_t time_receive_output_finish;
00129 int64_t total_bytes_received;
00130 int64_t total_bytes_sent;
00131 int64_t total_bytes_transferred;
00132 timestamp_t total_transfer_time;
00133 timestamp_t cmd_execution_time;
00134 int total_submissions;
00135 timestamp_t total_cmd_execution_time;
00137 double priority;
00139 int max_retries;
00141 struct rmsummary *resources_measured;
00142 struct rmsummary *resources_requested;
00144 category_allocation_t resource_request;
00146 char *category;
00148 timestamp_t time_app_delay;
00149 };
00150
00153 struct work_queue_stats {
00154 int total_workers_connected;
00155 int workers_init;
00156 int workers_idle;
00157 int workers_busy;
00159 int total_workers_joined;
00160 int total_workers_removed;
00161 int total_workers_lost;
00162 int total_workers_idled_out;
00163 int total_workers_fast_aborted;
00165
00166 int tasks_waiting;
00167 int tasks_running;
00168 int tasks_complete;
00170
00171 int total_tasks_dispatched;
00172 int total_tasks_complete;
00173 int total_tasks_failed;
00174 int total_tasks_cancelled;
00176 timestamp_t start_time;
00177 timestamp_t total_send_time;
00178 timestamp_t total_receive_time;
00179 timestamp_t total_good_transfer_time;
00181 timestamp_t total_execute_time;
00182 timestamp_t total_good_execute_time;
00185 int64_t total_bytes_sent;
00186 int64_t total_bytes_received;
00187 double efficiency;
00188 double idle_percentage;
00189 int capacity;
00191 double bandwidth;
00192 int64_t total_cores;
00193 int64_t total_memory;
00194 int64_t total_disk;
00195 int64_t total_gpus;
00196 int64_t committed_cores;
00197 int64_t committed_memory;
00198 int64_t committed_disk;
00199 int64_t committed_gpus;
00200 int64_t min_cores;
00201 int64_t max_cores;
00202 int64_t min_memory;
00203 int64_t max_memory;
00204 int64_t min_disk;
00205 int64_t max_disk;
00206 int64_t min_gpus;
00207 int64_t max_gpus;
00208 int port;
00209 int priority;
00210 int workers_ready;
00211 int workers_full;
00212 int total_worker_slots;
00213 int avg_capacity;
00214 };
00215
00216
00220
00228 struct work_queue_task *work_queue_task_create(const char *full_command);
00229
00235 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00236
00241 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00242
00259 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t flags);
00260
00275 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, work_queue_file_type_t type, work_queue_file_flags_t flags);
00276
00287 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, work_queue_file_flags_t);
00288
00302 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t, int recursive);
00303
00309 void work_queue_task_specify_max_retries( struct work_queue_task *t, int64_t max_retries );
00310
00316 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00317
00323 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00324
00330 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00331
00337 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00338
00346 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t useconds );
00347
00355 void work_queue_task_specify_running_time( struct work_queue_task *t, int64_t useconds );
00356
00363 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00364
00371 void work_queue_task_specify_category(struct work_queue_task *t, const char *category);
00372
00379 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00380
00387 void work_queue_task_specify_enviroment_variable( struct work_queue_task *t, const char *name, const char *value );
00388
00394 void work_queue_task_specify_algorithm(struct work_queue_task *t, work_queue_schedule_t algorithm);
00395
00400 void work_queue_task_delete(struct work_queue_task *t);
00401
00403
00407
00424 struct work_queue *work_queue_create(int port);
00425
00434 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00435
00436
00443 int work_queue_enable_monitoring_full(struct work_queue *q, char *monitor_output_directory);
00444
00453 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00454
00459 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00460
00468 void work_queue_blacklist_add_with_timeout(struct work_queue *q, const char *hostname, time_t seconds);
00469
00470
00475 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00476
00477
00481 void work_queue_blacklist_clear(struct work_queue *q);
00482
00496 void work_queue_invalidate_cached_file(struct work_queue *q, const char *local_name, work_queue_file_t type);
00497
00498
00513 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00514
00526 int work_queue_hungry(struct work_queue *q);
00527
00535 int work_queue_empty(struct work_queue *q);
00536
00543 int work_queue_port(struct work_queue *q);
00544
00549 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00550
00555 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00556
00562 void work_queue_get_stats_category(struct work_queue *q, const char *c, struct work_queue_stats *s);
00563
00564
00570 work_queue_task_state_t work_queue_task_state(struct work_queue *q, int taskid);
00571
00576 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00577
00582 double work_queue_get_effective_bandwidth(struct work_queue *q);
00583
00590 char * work_queue_get_worker_summary( struct work_queue *q );
00591
00601 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00602
00603
00613 int work_queue_activate_fast_abort_category(struct work_queue *q, const char *category, double multiplier);
00614
00615
00619 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00620
00626 void work_queue_specify_algorithm(struct work_queue *q, work_queue_schedule_t algorithm);
00627
00632 const char *work_queue_name(struct work_queue *q);
00633
00638 void work_queue_specify_name(struct work_queue *q, const char *name);
00639
00644 void work_queue_specify_priority(struct work_queue *q, int priority);
00645
00654 void work_queue_specify_num_tasks_left(struct work_queue *q, int ntasks);
00655
00661 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00662
00668 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00669
00675 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00676
00681 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00682
00687 int work_queue_shut_down_workers(struct work_queue *q, int n);
00688
00693 void work_queue_delete(struct work_queue *q);
00694
00700 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00701
00707 void work_queue_specify_password( struct work_queue *q, const char *password );
00708
00715 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00716
00721 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00722
00727 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00728
00734 void work_queue_master_preferred_connection(struct work_queue *q, const char *preferred_connection);
00735
00751 int work_queue_tune(struct work_queue *q, const char *name, double value);
00752
00759 void work_queue_specify_max_resources(struct work_queue *q, const struct rmsummary *rm);
00760
00768 void work_queue_specify_max_category_resources(struct work_queue *q, const char *category, const struct rmsummary *rm);
00769
00775 void work_queue_initialize_categories(struct work_queue *q, struct rmsummary *max, const char *summaries_file);
00776
00777
00779
00783
00784 #define WORK_QUEUE_TASK_ORDER_FIFO 0
00785 #define WORK_QUEUE_TASK_ORDER_LIFO 1
00793 void work_queue_specify_task_order(struct work_queue *q, int order);
00794
00795
00796 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0
00797 #define WORK_QUEUE_MASTER_MODE_CATALOG 1
00806 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00807
00808
00814 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00815
00824 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00825
00833 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00834
00842 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00843
00851 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00852
00860 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00861
00863
00864
00865
00866
00867
00868
00869
00870
00871 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00872
00873 #endif