work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "rmsummary.h"
00023 
00024 #define WORK_QUEUE_DEFAULT_PORT 9123  
00025 #define WORK_QUEUE_RANDOM_PORT  0    
00026 #define WORK_QUEUE_WAITFORTASK  -1    
00028 #define WORK_QUEUE_SCHEDULE_UNSET 0
00029 #define WORK_QUEUE_SCHEDULE_FCFS         1 
00030 #define WORK_QUEUE_SCHEDULE_FILES        2 
00031 #define WORK_QUEUE_SCHEDULE_TIME         3 
00032 #define WORK_QUEUE_SCHEDULE_RAND         4 
00034 #define WORK_QUEUE_INPUT  0     
00035 #define WORK_QUEUE_OUTPUT 1     
00037 #define WORK_QUEUE_NOCACHE 0    
00038 #define WORK_QUEUE_CACHE 1      
00039 #define WORK_QUEUE_SYMLINK 2    /* Create a symlink to the file rather than copying it, if possible. */
00040 #define WORK_QUEUE_PREEXIST 4   /* If the filename already exists on the host, use it in place. */
00041 #define WORK_QUEUE_THIRDGET 8   /* Access the file on the client from a shared filesystem */
00042 #define WORK_QUEUE_THIRDPUT 8   /* Access the file on the client from a shared filesystem (included for readability) */
00043 #define WORK_QUEUE_WATCH 16     
00045 #define WORK_QUEUE_RESET_ALL        0  
00046 #define WORK_QUEUE_RESET_KEEP_TASKS 1  
00048 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30    
00051 #define WORK_QUEUE_RESULT_SUCCESS 0                
00052 #define WORK_QUEUE_RESULT_INPUT_MISSING 1  
00053 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 
00054 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4 
00055 #define WORK_QUEUE_RESULT_SIGNAL         8 
00056 #define WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 
00057 #define WORK_QUEUE_RESULT_TASK_TIMEOUT 32 
00059 extern double wq_option_fast_abort_multiplier; 
00061 extern int wq_option_scheduler; 
00065 struct work_queue_task {
00066         char *tag;                      
00067         char *command_line;             
00068         int worker_selection_algorithm;           
00069         char *output;                   
00070         struct list *input_files;       
00071         struct list *output_files;      
00072         int taskid;                     
00073         int return_status;              
00074         int result;                     
00075         char *host;                     
00076         char *hostname;                 
00078         timestamp_t time_committed;     
00080         timestamp_t time_task_submit;   
00081         timestamp_t time_task_finish;   
00082         timestamp_t time_send_input_start;      
00083         timestamp_t time_send_input_finish;     
00084         timestamp_t time_execute_cmd_start;                 
00085         timestamp_t time_execute_cmd_finish;            
00086         timestamp_t time_receive_result_start;  
00087         timestamp_t time_receive_result_finish; 
00088         timestamp_t time_receive_output_start;  
00089         timestamp_t time_receive_output_finish; 
00091         int64_t total_bytes_received;
00092         int64_t total_bytes_sent;
00093         int64_t total_bytes_transferred;
00094         timestamp_t total_transfer_time;    
00095         timestamp_t cmd_execution_time;    
00096         int total_submissions;                     
00097         timestamp_t total_cmd_execution_time;   
00099         int64_t maximum_end_time;                       
00100         int64_t memory;                       
00101         int64_t disk;
00102         int cores;
00103         int gpus;
00104         int unlabeled;
00105         struct rmsummary *resources_measured;   
00107         double priority;                        
00109         timestamp_t time_app_delay;      
00111 };
00112 
00115 struct work_queue_stats {
00116         int total_workers_connected;    
00117         int workers_init;               
00118         int workers_idle;               
00119         int workers_busy;               
00120         int total_workers_joined;       
00121         int total_workers_removed;      
00123         int tasks_waiting;              
00124         int tasks_running;              
00125         int tasks_complete;             
00126         int total_tasks_dispatched;     
00127         int total_tasks_complete;       
00128         int total_tasks_failed;         
00129         int total_tasks_cancelled;      
00131         timestamp_t start_time;         
00132         timestamp_t total_send_time;    
00133         timestamp_t total_receive_time; 
00134         timestamp_t total_good_transfer_time;    
00136         timestamp_t total_execute_time; 
00137         timestamp_t total_good_execute_time; 
00140         int64_t total_bytes_sent;       
00141         int64_t total_bytes_received;   
00142         double efficiency;              
00143         double idle_percentage;         
00144         int capacity;                   
00146         double  bandwidth;              
00147         int64_t total_cores;            
00148         int64_t total_memory;           
00149         int64_t total_disk;                 
00150         int64_t total_gpus;             
00151         int64_t committed_cores;        
00152         int64_t committed_memory;       
00153         int64_t committed_disk;         
00154         int64_t committed_gpus;         
00155         int64_t min_cores;              
00156         int64_t max_cores;              
00157         int64_t min_memory;             
00158         int64_t max_memory;             
00159         int64_t min_disk;               
00160         int64_t max_disk;               
00161         int64_t min_gpus;               
00162         int64_t max_gpus;               
00163         int port;                                               
00164         int priority;                                   
00165         int workers_ready;              
00166         int workers_full;               
00167         int total_worker_slots;         
00168         int avg_capacity;               
00169 };
00170 
00171 
00175 
00183 struct work_queue_task *work_queue_task_create(const char *full_command);
00184 
00190 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00191 
00196 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00197 
00214 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00215 
00230 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00231 
00242 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00243 
00257 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00258 
00264 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00265 
00271 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00272 
00278 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00279 
00285 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00286 
00292 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00293 
00300 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00301 
00308 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00309 
00319 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00320 
00325 void work_queue_task_delete(struct work_queue_task *t);
00326 
00328 
00332 
00349 struct work_queue *work_queue_create(int port);
00350 
00358 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00359 
00368 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00369 
00374 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00375 
00376 
00381 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00382 
00383 
00387 void work_queue_blacklist_clear(struct work_queue *q);
00388 
00403 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00404 
00416 int work_queue_hungry(struct work_queue *q);
00417 
00425 int work_queue_empty(struct work_queue *q);
00426 
00433 int work_queue_port(struct work_queue *q);
00434 
00439 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00440 
00445 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00446 
00451 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00452 
00457 double work_queue_get_effective_bandwidth(struct work_queue *q);
00458 
00465 char * work_queue_get_worker_summary( struct work_queue *q );
00466 
00472 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00473 
00474 
00478 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00479 
00489 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00490 
00495 const char *work_queue_name(struct work_queue *q);
00496 
00501 void work_queue_specify_name(struct work_queue *q, const char *name);
00502 
00507 void work_queue_specify_priority(struct work_queue *q, int priority);
00508 
00514 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00515 
00521 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00522 
00528 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00529 
00534 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00535 
00540 int work_queue_shut_down_workers(struct work_queue *q, int n);
00541 
00546 void work_queue_delete(struct work_queue *q);
00547 
00553 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00554 
00560 void work_queue_specify_password( struct work_queue *q, const char *password );
00561 
00568 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00569 
00574 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00575 
00580 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00581 
00582 
00598 int work_queue_tune(struct work_queue *q, const char *name, double value);
00599 
00601 
00605 
00606 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00607 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00615 void work_queue_specify_task_order(struct work_queue *q, int order);
00616 
00617 
00618 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00619 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00628 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00629 
00635 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00636 
00645 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00646 
00654 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00655 
00663 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00664 
00672 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00673 
00681 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00682 
00684 
00685 /* Experimental feature - intentionally left undocumented.
00686 This feature exists to simplify performance evaulation and is not recommended
00687 for production use since it delays execution of the workload. 
00688 Force the master to wait for the given number of workers to connect before
00689 starting to dispatch tasks.  
00690 @param q A work queue object.
00691 @param worker The number of workers to wait before tasks are dispatched.*/
00692 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00693 
00694 #endif

Generated on 17 Feb 2015 for cctools by  doxygen 1.6.1