work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "rmsummary.h"
00023 
00024 #define WORK_QUEUE_DEFAULT_PORT 9123               
00025 #define WORK_QUEUE_RANDOM_PORT  0                  
00027 #define WORK_QUEUE_WAITFORTASK  -1                 
00029 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00030 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT  30   
00032 typedef enum {
00033         WORK_QUEUE_INPUT  = 0,                         
00034         WORK_QUEUE_OUTPUT = 1                          
00035 } work_queue_file_type_t;
00036 
00037 typedef enum {
00038         WORK_QUEUE_NOCACHE  = 0, 
00039         WORK_QUEUE_CACHE    = 1, 
00040         WORK_QUEUE_SYMLINK  = 2, 
00041         WORK_QUEUE_PREEXIST = 4, 
00042         WORK_QUEUE_THIRDGET = 8, 
00043         WORK_QUEUE_THIRDPUT = 8, 
00044         WORK_QUEUE_WATCH    = 16 
00045 } work_queue_file_flags_t;
00046 
00047 typedef enum {
00048         WORK_QUEUE_SCHEDULE_UNSET = 0,
00049         WORK_QUEUE_SCHEDULE_FCFS,      
00050         WORK_QUEUE_SCHEDULE_FILES,     
00051         WORK_QUEUE_SCHEDULE_TIME,      
00052         WORK_QUEUE_SCHEDULE_RAND,      
00053         WORK_QUEUE_SCHEDULE_WORST      
00054 } work_queue_schedule_t;
00055 
00056 
00057 typedef enum {
00058         WORK_QUEUE_RESULT_SUCCESS        = 0,       
00059         WORK_QUEUE_RESULT_INPUT_MISSING  = 1,       
00060         WORK_QUEUE_RESULT_OUTPUT_MISSING = 2,       
00061         WORK_QUEUE_RESULT_STDOUT_MISSING = 4,       
00062         WORK_QUEUE_RESULT_SIGNAL         = 8,       
00063         WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION = 16, 
00064         WORK_QUEUE_RESULT_TASK_TIMEOUT   = 32       
00065 } work_queue_result_t;
00066 
00067 typedef enum {
00068         WORK_QUEUE_TASK_UNKNOWN = 0,       
00069         WORK_QUEUE_TASK_READY,             
00070         WORK_QUEUE_TASK_RUNNING,           
00071         WORK_QUEUE_TASK_WAITING_RETRIEVAL, 
00072         WORK_QUEUE_TASK_RETRIEVED,         
00073         WORK_QUEUE_TASK_DONE,              
00074         WORK_QUEUE_TASK_CANCELED           
00075 } work_queue_task_state_t;
00076 
00077 extern double wq_option_fast_abort_multiplier; 
00083 extern int wq_option_scheduler;                
00093 struct work_queue_task {
00094         char *tag;                                             
00095         char *command_line;                                    
00096         work_queue_schedule_t worker_selection_algorithm; 
00097         char *output;                                          
00098         struct list *input_files;                              
00099         struct list *output_files;                             
00100         struct list *env_list;                                 
00101         int taskid;                                            
00102         int return_status;                                     
00103         work_queue_result_t result;                       
00104         char *host;                                            
00105         char *hostname;                                        
00107         timestamp_t time_committed;                            
00109         timestamp_t time_task_submit;                          
00110         timestamp_t time_task_finish;                          
00111         timestamp_t time_send_input_start;                     
00112         timestamp_t time_send_input_finish;                    
00113         timestamp_t time_execute_cmd_start;                    
00114         timestamp_t time_execute_cmd_finish;                   
00115         timestamp_t time_receive_result_start;                 
00116         timestamp_t time_receive_result_finish;                
00117         timestamp_t time_receive_output_start;                 
00118         timestamp_t time_receive_output_finish;                
00120         int64_t total_bytes_received;                          
00121         int64_t total_bytes_sent;                              
00122         int64_t total_bytes_transferred;                       
00123         timestamp_t total_transfer_time;                       
00124         timestamp_t cmd_execution_time;                        
00125         int total_submissions;                                 
00126         timestamp_t total_cmd_execution_time;                  
00128         int64_t maximum_end_time;                              
00129         int64_t memory;                                        
00130         int64_t disk;                                          
00131         int cores;                                             
00132         int gpus;                                              
00133         int unlabeled;                                         
00135         double priority;                                       
00137         struct rmsummary *resources_measured;                  
00139         timestamp_t time_app_delay;                            
00141 };
00142 
00145 struct work_queue_stats {
00146         int total_workers_connected;    
00147         int workers_init;               
00148         int workers_idle;               
00149         int workers_busy;               
00150         int total_workers_joined;       
00151         int total_workers_removed;      
00153         int tasks_waiting;              
00154         int tasks_running;              
00155         int tasks_complete;             
00156         int total_tasks_dispatched;     
00157         int total_tasks_complete;       
00158         int total_tasks_failed;         
00159         int total_tasks_cancelled;      
00161         timestamp_t start_time;         
00162         timestamp_t total_send_time;    
00163         timestamp_t total_receive_time; 
00164         timestamp_t total_good_transfer_time;    
00166         timestamp_t total_execute_time;      
00167         timestamp_t total_good_execute_time; 
00170         int64_t total_bytes_sent;       
00171         int64_t total_bytes_received;   
00172         double efficiency;              
00173         double idle_percentage;         
00174         int capacity;                   
00176         double  bandwidth;              
00177         int64_t total_cores;            
00178         int64_t total_memory;           
00179         int64_t total_disk;                 
00180         int64_t total_gpus;             
00181         int64_t committed_cores;        
00182         int64_t committed_memory;       
00183         int64_t committed_disk;         
00184         int64_t committed_gpus;         
00185         int64_t min_cores;              
00186         int64_t max_cores;              
00187         int64_t min_memory;             
00188         int64_t max_memory;             
00189         int64_t min_disk;               
00190         int64_t max_disk;               
00191         int64_t min_gpus;               
00192         int64_t max_gpus;               
00193         int port;
00194         int priority;
00195         int workers_ready;              
00196         int workers_full;               
00197         int total_worker_slots;         
00198         int avg_capacity;               
00199 };
00200 
00201 
00205 
00213 struct work_queue_task *work_queue_task_create(const char *full_command);
00214 
00220 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00221 
00226 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00227 
00244 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t flags);
00245 
00260 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, work_queue_file_type_t type, work_queue_file_flags_t flags);
00261 
00272 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, work_queue_file_flags_t);
00273 
00287 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t, int recursive);
00288 
00294 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00295 
00301 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00302 
00308 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00309 
00315 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00316 
00322 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00323 
00330 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00331 
00338 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00339 
00346 void work_queue_task_specify_enviroment_variable( struct work_queue_task *t, const char *name, const char *value );
00347 
00353 void work_queue_task_specify_algorithm(struct work_queue_task *t, work_queue_schedule_t algorithm);
00354 
00359 void work_queue_task_delete(struct work_queue_task *t);
00360 
00362 
00366 
00383 struct work_queue *work_queue_create(int port);
00384 
00392 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00393 
00402 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00403 
00408 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00409 
00410 
00415 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00416 
00417 
00421 void work_queue_blacklist_clear(struct work_queue *q);
00422 
00437 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00438 
00450 int work_queue_hungry(struct work_queue *q);
00451 
00459 int work_queue_empty(struct work_queue *q);
00460 
00467 int work_queue_port(struct work_queue *q);
00468 
00473 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00474 
00479 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00480 
00481 
00487 work_queue_task_state_t work_queue_task_state(struct work_queue *q, int taskid);
00488 
00493 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00494 
00499 double work_queue_get_effective_bandwidth(struct work_queue *q);
00500 
00507 char * work_queue_get_worker_summary( struct work_queue *q );
00508 
00514 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00515 
00516 
00520 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00521 
00527 void work_queue_specify_algorithm(struct work_queue *q, work_queue_schedule_t algorithm);
00528 
00533 const char *work_queue_name(struct work_queue *q);
00534 
00539 void work_queue_specify_name(struct work_queue *q, const char *name);
00540 
00545 void work_queue_specify_priority(struct work_queue *q, int priority);
00546 
00552 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00553 
00559 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00560 
00566 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00567 
00572 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00573 
00578 int work_queue_shut_down_workers(struct work_queue *q, int n);
00579 
00584 void work_queue_delete(struct work_queue *q);
00585 
00591 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00592 
00598 void work_queue_specify_password( struct work_queue *q, const char *password );
00599 
00606 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00607 
00612 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00613 
00618 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00619 
00620 
00636 int work_queue_tune(struct work_queue *q, const char *name, double value);
00637 
00639 
00643 
00644 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00645 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00653 void work_queue_specify_task_order(struct work_queue *q, int order);
00654 
00655 
00656 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00657 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00666 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00667 
00673 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00674 
00683 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00684 
00692 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00693 
00701 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00702 
00710 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00711 
00719 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00720 
00722 
00723 /* Experimental feature - intentionally left undocumented.
00724 This feature exists to simplify performance evaulation and is not recommended
00725 for production use since it delays execution of the workload.
00726 Force the master to wait for the given number of workers to connect before
00727 starting to dispatch tasks.
00728 @param q A work queue object.
00729 @param worker The number of workers to wait before tasks are dispatched.*/
00730 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00731 
00732 #endif

Generated on 18 Aug 2015 for cctools by  doxygen 1.6.1