work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 
00023 #define WORK_QUEUE_DEFAULT_PORT 9123  
00024 #define WORK_QUEUE_RANDOM_PORT  0    
00025 #define WORK_QUEUE_WAITFORTASK  -1    
00027 #define WORK_QUEUE_SCHEDULE_UNSET 0
00028 #define WORK_QUEUE_SCHEDULE_FCFS         1 
00029 #define WORK_QUEUE_SCHEDULE_FILES        2 
00030 #define WORK_QUEUE_SCHEDULE_TIME         3 
00031 #define WORK_QUEUE_SCHEDULE_RAND         4 
00033 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00034 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00036 #define WORK_QUEUE_INPUT  0     
00037 #define WORK_QUEUE_OUTPUT 1     
00039 #define WORK_QUEUE_NOCACHE 0    
00040 #define WORK_QUEUE_CACHE 1      
00041 #define WORK_QUEUE_SYMLINK 2    /* Create a symlink to the file rather than copying it, if possible. */
00042 #define WORK_QUEUE_PREEXIST 4   /* If the filename already exists on the host, use it in place. */
00043 #define WORK_QUEUE_THIRDGET 8   /* Access the file on the client from a shared filesystem */
00044 #define WORK_QUEUE_THIRDPUT 8   /* Access the file on the client from a shared filesystem (included for readability) */
00045 
00046 #define WORK_QUEUE_RESET_ALL        0  
00047 #define WORK_QUEUE_RESET_KEEP_TASKS 1  
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00050 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30    
00052 #define WORK_QUEUE_RESULT_SUCCESS 0                
00053 #define WORK_QUEUE_RESULT_INPUT_MISSING 1  
00054 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 
00055 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4 
00057 extern double wq_option_fast_abort_multiplier; 
00059 extern int wq_option_scheduler; 
00063 struct work_queue_task {
00064         char *tag;                      
00065         char *command_line;             
00066         int worker_selection_algorithm;           
00067         char *output;                   
00068         struct list *input_files;       
00069         struct list *output_files;      
00070         int taskid;                     
00071         int return_status;              
00072         int result;                     
00073         char *host;                     
00074         char *hostname;                 
00076         timestamp_t time_committed;     
00078         timestamp_t time_task_submit;   
00079         timestamp_t time_task_finish;   
00080         timestamp_t time_send_input_start;      
00081         timestamp_t time_send_input_finish;     
00082         timestamp_t time_execute_cmd_start;                 
00083         timestamp_t time_execute_cmd_finish;            
00084         timestamp_t time_receive_result_start;  
00085         timestamp_t time_receive_result_finish; 
00086         timestamp_t time_receive_output_start;  
00087         timestamp_t time_receive_output_finish; 
00089         int64_t total_bytes_received;
00090         int64_t total_bytes_sent;
00091         int64_t total_bytes_transferred;
00092         timestamp_t total_transfer_time;    
00093         timestamp_t cmd_execution_time;    
00094         int total_submissions;                     
00095         timestamp_t total_cmd_execution_time;   
00097         int64_t memory;                       
00098         int64_t disk;
00099         int cores;
00100         int gpus;
00101         int unlabeled;
00102         
00103         timestamp_t time_app_delay;      
00104 };
00105 
00108 struct work_queue_stats {
00109         int total_workers_connected;    
00110         int workers_init;               
00111         int workers_idle;               
00112         int workers_busy;               
00113         int total_workers_joined;       
00114         int total_workers_removed;      
00116         int tasks_waiting;              
00117         int tasks_running;              
00118         int tasks_complete;             
00119         int total_tasks_dispatched;     
00120         int total_tasks_complete;       
00121         int total_tasks_cancelled;      
00123         timestamp_t start_time;         
00124         timestamp_t total_send_time;    
00125         timestamp_t total_receive_time; 
00126         int64_t total_bytes_sent;       
00127         int64_t total_bytes_received;   
00128         double efficiency;              
00129         double idle_percentage;         
00130         int capacity;                   
00132         double  bandwidth;              
00133         int64_t total_cores;            
00134         int64_t total_memory;           
00135         int64_t total_disk;                 
00136         int64_t total_gpus;             
00137         int64_t committed_cores;        
00138         int64_t committed_memory;       
00139         int64_t committed_disk;         
00140         int64_t committed_gpus;         
00141         int64_t min_cores;              
00142         int64_t max_cores;              
00143         int64_t min_memory;             
00144         int64_t max_memory;             
00145         int64_t min_disk;               
00146         int64_t max_disk;               
00147         int64_t min_gpus;               
00148         int64_t max_gpus;               
00149         timestamp_t total_execute_time; 
00150         timestamp_t total_good_execute_time; 
00152         int port;                                               
00153         int priority;                                   
00154         int workers_ready;              
00155         int workers_full;               
00156         int total_worker_slots;         
00157         int avg_capacity;               
00158 };
00159 
00160 
00164 
00172 struct work_queue_task *work_queue_task_create(const char *full_command);
00173 
00179 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00180 
00185 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00186 
00199 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00200 
00215 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00216 
00227 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00228 
00242 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00243 
00249 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00250 
00256 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00257 
00263 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00264 
00270 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00271 
00278 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00279 
00289 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00290 
00295 void work_queue_task_delete(struct work_queue_task *t);
00296 
00298 
00302 
00319 struct work_queue *work_queue_create(int port);
00320 
00328 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00329 
00338 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00339 
00354 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00355 
00367 int work_queue_hungry(struct work_queue *q);
00368 
00376 int work_queue_empty(struct work_queue *q);
00377 
00384 int work_queue_port(struct work_queue *q);
00385 
00390 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00391 
00396 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00397 
00402 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00403 
00408 double work_queue_get_effective_bandwidth(struct work_queue *q);
00409 
00416 char * work_queue_get_worker_summary( struct work_queue *q );
00417 
00423 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00424 
00425 
00437 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00438 
00448 void work_queue_specify_task_order(struct work_queue *q, int order);
00449 
00454 const char *work_queue_name(struct work_queue *q);
00455 
00460 void work_queue_specify_name(struct work_queue *q, const char *name);
00461 
00466 void work_queue_specify_priority(struct work_queue *q, int priority);
00467 
00473 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00474 
00480 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00481 
00487 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00488 
00493 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00494 
00501 void work_queue_reset(struct work_queue *q, int flags);
00502 
00507 int work_queue_shut_down_workers(struct work_queue *q, int n);
00508 
00513 void work_queue_delete(struct work_queue *q);
00514 
00520 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00521 
00527 void work_queue_specify_password( struct work_queue *q, const char *password );
00528 
00535 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00536 
00541 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00542 
00547 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00548 
00549 
00565 int work_queue_tune(struct work_queue *q, const char *name, double value);
00566 
00568 
00572 
00573 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00574 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00583 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00584 
00590 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00591 
00600 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00601 
00609 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00610 
00618 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00619 
00627 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00628 
00636 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00637 
00639 
00640 /* Experimental feature - intentionally left undocumented.
00641 This feature exists to simplify performance evaulation and is not recommended
00642 for production use since it delays execution of the workload. 
00643 Force the master to wait for the given number of workers to connect before
00644 starting to dispatch tasks.  
00645 @param q A work queue object.
00646 @param worker The number of workers to wait before tasks are dispatched.*/
00647 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00648 
00649 #endif

Generated on 27 Aug 2014 for cctools by  doxygen 1.4.7