work_queue.h

Go to the documentation of this file.
00001 /*
00002    Copyright (C) 2008- The University of Notre Dame
00003    This software is distributed under the GNU General Public License.
00004    See the file COPYING for details.
00005    */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "rmsummary.h"
00023 
00024 #define WORK_QUEUE_DEFAULT_PORT 9123               
00025 #define WORK_QUEUE_RANDOM_PORT  0                  
00027 #define WORK_QUEUE_WAITFORTASK  -1                 
00029 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00030 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT  30   
00032 typedef enum {
00033         WORK_QUEUE_INPUT  = 0,                         
00034         WORK_QUEUE_OUTPUT = 1                          
00035 } work_queue_file_type_t;
00036 
00037 typedef enum {
00038         WORK_QUEUE_NOCACHE  = 0, 
00039         WORK_QUEUE_CACHE    = 1, 
00040         WORK_QUEUE_SYMLINK  = 2, 
00041         WORK_QUEUE_PREEXIST = 4, 
00042         WORK_QUEUE_THIRDGET = 8, 
00043         WORK_QUEUE_THIRDPUT = 8, 
00044         WORK_QUEUE_WATCH    = 16 
00045 } work_queue_file_flags_t;
00046 
00047 typedef enum {
00048         WORK_QUEUE_SCHEDULE_UNSET = 0,
00049         WORK_QUEUE_SCHEDULE_FCFS,      
00050         WORK_QUEUE_SCHEDULE_FILES,     
00051         WORK_QUEUE_SCHEDULE_TIME,      
00052         WORK_QUEUE_SCHEDULE_RAND,      
00053         WORK_QUEUE_SCHEDULE_WORST      
00054 } work_queue_schedule_t;
00055 
00056 
00057 typedef enum {
00058         WORK_QUEUE_RESULT_SUCCESS        = 0,       
00059         WORK_QUEUE_RESULT_INPUT_MISSING  = 1,       
00060         WORK_QUEUE_RESULT_OUTPUT_MISSING = 2,       
00061         WORK_QUEUE_RESULT_STDOUT_MISSING = 4,       
00062         WORK_QUEUE_RESULT_SIGNAL         = 8,       
00063         WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION = 16, 
00064         WORK_QUEUE_RESULT_TASK_TIMEOUT   = 32       
00065 } work_queue_result_t;
00066 
00067 typedef enum {
00068         WORK_QUEUE_TASK_UNKNOWN = 0,       
00069         WORK_QUEUE_TASK_READY,             
00070         WORK_QUEUE_TASK_RUNNING,           
00071         WORK_QUEUE_TASK_WAITING_RETRIEVAL, 
00072         WORK_QUEUE_TASK_RETRIEVED,         
00073         WORK_QUEUE_TASK_DONE,              
00074         WORK_QUEUE_TASK_CANCELED,          
00075         WORK_QUEUE_TASK_WAITING_RESUBMISSION 
00076 } work_queue_task_state_t;
00077 
00078 extern double wq_option_fast_abort_multiplier; 
00084 extern int wq_option_scheduler;                
00094 struct work_queue_task {
00095         char *tag;                                             
00096         char *command_line;                                    
00097         work_queue_schedule_t worker_selection_algorithm; 
00098         char *output;                                          
00099         struct list *input_files;                              
00100         struct list *output_files;                             
00101         struct list *env_list;                                 
00102         int taskid;                                            
00103         int return_status;                                     
00104         work_queue_result_t result;                       
00105         char *host;                                            
00106         char *hostname;                                        
00108         timestamp_t time_committed;                            
00110         timestamp_t time_task_submit;                          
00111         timestamp_t time_task_finish;                          
00112         timestamp_t time_send_input_start;                     
00113         timestamp_t time_send_input_finish;                    
00114         timestamp_t time_execute_cmd_start;                    
00115         timestamp_t time_execute_cmd_finish;                   
00116         timestamp_t time_receive_result_start;                 
00117         timestamp_t time_receive_result_finish;                
00118         timestamp_t time_receive_output_start;                 
00119         timestamp_t time_receive_output_finish;                
00121         int64_t total_bytes_received;                          
00122         int64_t total_bytes_sent;                              
00123         int64_t total_bytes_transferred;                       
00124         timestamp_t total_transfer_time;                       
00125         timestamp_t cmd_execution_time;                        
00126         int total_submissions;                                 
00127         timestamp_t total_cmd_execution_time;                  
00129         int64_t maximum_end_time;                              
00130         int64_t memory;                                        
00131         int64_t disk;                                          
00132         int cores;                                             
00133         int gpus;                                              
00134         int unlabeled;                                         
00136         double priority;                                       
00138         struct rmsummary *resources_measured;                  
00140         timestamp_t time_app_delay;                            
00142 };
00143 
00146 struct work_queue_stats {
00147         int total_workers_connected;    
00148         int workers_init;               
00149         int workers_idle;               
00150         int workers_busy;               
00151         int total_workers_joined;       
00152         int total_workers_removed;      
00154         int tasks_waiting;              
00155         int tasks_running;              
00156         int tasks_complete;             
00157         int total_tasks_dispatched;     
00158         int total_tasks_complete;       
00159         int total_tasks_failed;         
00160         int total_tasks_cancelled;      
00162         timestamp_t start_time;         
00163         timestamp_t total_send_time;    
00164         timestamp_t total_receive_time; 
00165         timestamp_t total_good_transfer_time;    
00167         timestamp_t total_execute_time;      
00168         timestamp_t total_good_execute_time; 
00171         int64_t total_bytes_sent;       
00172         int64_t total_bytes_received;   
00173         double efficiency;              
00174         double idle_percentage;         
00175         int capacity;                   
00177         double  bandwidth;              
00178         int64_t total_cores;            
00179         int64_t total_memory;           
00180         int64_t total_disk;                 
00181         int64_t total_gpus;             
00182         int64_t committed_cores;        
00183         int64_t committed_memory;       
00184         int64_t committed_disk;         
00185         int64_t committed_gpus;         
00186         int64_t min_cores;              
00187         int64_t max_cores;              
00188         int64_t min_memory;             
00189         int64_t max_memory;             
00190         int64_t min_disk;               
00191         int64_t max_disk;               
00192         int64_t min_gpus;               
00193         int64_t max_gpus;               
00194         int port;
00195         int priority;
00196         int workers_ready;              
00197         int workers_full;               
00198         int total_worker_slots;         
00199         int avg_capacity;               
00200 };
00201 
00202 
00206 
00214 struct work_queue_task *work_queue_task_create(const char *full_command);
00215 
00221 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00222 
00227 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00228 
00245 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t flags);
00246 
00261 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, work_queue_file_type_t type, work_queue_file_flags_t flags);
00262 
00273 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, work_queue_file_flags_t);
00274 
00288 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t, int recursive);
00289 
00295 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00296 
00302 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00303 
00309 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00310 
00316 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00317 
00323 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00324 
00331 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00332 
00339 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00340 
00347 void work_queue_task_specify_enviroment_variable( struct work_queue_task *t, const char *name, const char *value );
00348 
00354 void work_queue_task_specify_algorithm(struct work_queue_task *t, work_queue_schedule_t algorithm);
00355 
00360 void work_queue_task_delete(struct work_queue_task *t);
00361 
00363 
00367 
00384 struct work_queue *work_queue_create(int port);
00385 
00393 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00394 
00403 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00404 
00409 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00410 
00411 
00416 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00417 
00418 
00422 void work_queue_blacklist_clear(struct work_queue *q);
00423 
00438 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00439 
00451 int work_queue_hungry(struct work_queue *q);
00452 
00460 int work_queue_empty(struct work_queue *q);
00461 
00468 int work_queue_port(struct work_queue *q);
00469 
00474 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00475 
00480 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00481 
00482 
00488 work_queue_task_state_t work_queue_task_state(struct work_queue *q, int taskid);
00489 
00494 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00495 
00500 double work_queue_get_effective_bandwidth(struct work_queue *q);
00501 
00508 char * work_queue_get_worker_summary( struct work_queue *q );
00509 
00515 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00516 
00517 
00521 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00522 
00528 void work_queue_specify_algorithm(struct work_queue *q, work_queue_schedule_t algorithm);
00529 
00534 const char *work_queue_name(struct work_queue *q);
00535 
00540 void work_queue_specify_name(struct work_queue *q, const char *name);
00541 
00546 void work_queue_specify_priority(struct work_queue *q, int priority);
00547 
00556 void work_queue_specify_num_tasks_left(struct work_queue *q, int ntasks);
00557 
00563 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00564 
00570 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00571 
00577 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00578 
00583 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00584 
00589 int work_queue_shut_down_workers(struct work_queue *q, int n);
00590 
00595 void work_queue_delete(struct work_queue *q);
00596 
00602 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00603 
00609 void work_queue_specify_password( struct work_queue *q, const char *password );
00610 
00617 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00618 
00623 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00624 
00629 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00630 
00636 void work_queue_master_preferred_connection(struct work_queue *q, const char *preferred_connection);
00637 
00653 int work_queue_tune(struct work_queue *q, const char *name, double value);
00654 
00656 
00660 
00661 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00662 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00670 void work_queue_specify_task_order(struct work_queue *q, int order);
00671 
00672 
00673 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00674 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00683 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00684 
00685 
00691 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00692 
00701 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00702 
00710 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00711 
00719 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00720 
00728 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00729 
00737 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00738 
00740 
00741 /* Experimental feature - intentionally left undocumented.
00742    This feature exists to simplify performance evaulation and is not recommended
00743    for production use since it delays execution of the workload.
00744    Force the master to wait for the given number of workers to connect before
00745    starting to dispatch tasks.
00746    @param q A work queue object.
00747    @param worker The number of workers to wait before tasks are dispatched.*/
00748 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00749 
00750 #endif
00751 
00752 
00753 /* vim: set noexpandtab tabstop=4: */

Generated on 7 Oct 2015 for cctools by  doxygen 1.6.1