work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "category.h"
00023 #include "rmsummary.h"
00024 
00025 #define WORK_QUEUE_DEFAULT_PORT 9123               
00026 #define WORK_QUEUE_RANDOM_PORT  0                  
00028 #define WORK_QUEUE_WAITFORTASK  -1                 
00030 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 120  
00031 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT  30   
00033 typedef enum {
00034         WORK_QUEUE_INPUT  = 0,                         
00035         WORK_QUEUE_OUTPUT = 1                          
00036 } work_queue_file_type_t;
00037 
00038 typedef enum {
00039         WORK_QUEUE_NOCACHE  = 0, 
00040         WORK_QUEUE_CACHE    = 1, 
00041         WORK_QUEUE_SYMLINK  = 2, 
00042         WORK_QUEUE_PREEXIST = 4, 
00043         WORK_QUEUE_THIRDGET = 8, 
00044         WORK_QUEUE_THIRDPUT = 8, 
00045         WORK_QUEUE_WATCH    = 16 
00046 } work_queue_file_flags_t;
00047 
00048 typedef enum {
00049         WORK_QUEUE_SCHEDULE_UNSET = 0,
00050         WORK_QUEUE_SCHEDULE_FCFS,      
00051         WORK_QUEUE_SCHEDULE_FILES,     
00052         WORK_QUEUE_SCHEDULE_TIME,      
00053         WORK_QUEUE_SCHEDULE_RAND,      
00054         WORK_QUEUE_SCHEDULE_WORST      
00055 } work_queue_schedule_t;
00056 
00057 
00058 typedef enum {
00059         WORK_QUEUE_RESULT_SUCCESS             = 0,      
00060         WORK_QUEUE_RESULT_INPUT_MISSING       = 1,      
00061         WORK_QUEUE_RESULT_OUTPUT_MISSING      = 2,      
00062         WORK_QUEUE_RESULT_STDOUT_MISSING      = 4,      
00063         WORK_QUEUE_RESULT_SIGNAL              = 1 << 3, 
00064         WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION = 2 << 3, 
00065         WORK_QUEUE_RESULT_TASK_TIMEOUT        = 3 << 3, 
00066         WORK_QUEUE_RESULT_UNKNOWN             = 4 << 3, 
00067         WORK_QUEUE_RESULT_FORSAKEN            = 5 << 3, 
00068         WORK_QUEUE_RESULT_MAX_RETRIES         = 6 << 3, 
00069         WORK_QUEUE_RESULT_TASK_MAX_RUN_TIME   = 7 << 3, 
00070 } work_queue_result_t;
00071 
00072 typedef enum {
00073         WORK_QUEUE_TASK_UNKNOWN = 0,       
00074         WORK_QUEUE_TASK_READY,             
00075         WORK_QUEUE_TASK_RUNNING,           
00076         WORK_QUEUE_TASK_WAITING_RETRIEVAL, 
00077         WORK_QUEUE_TASK_RETRIEVED,         
00078         WORK_QUEUE_TASK_DONE,              
00079         WORK_QUEUE_TASK_CANCELED,           
00080         WORK_QUEUE_TASK_WAITING_RESUBMISSION 
00081 } work_queue_task_state_t;
00082 
00083 typedef enum {
00084         WORK_QUEUE_FILE = 1,              
00085         WORK_QUEUE_BUFFER,                
00086         WORK_QUEUE_REMOTECMD,             
00087         WORK_QUEUE_FILE_PIECE,            
00088         WORK_QUEUE_DIRECTORY,             
00089         WORK_QUEUE_URL                    
00090 } work_queue_file_t;
00091 
00092 extern int wq_option_scheduler;                
00102 struct work_queue_task {
00103         char *tag;                                             
00104         char *command_line;                                    
00105         work_queue_schedule_t worker_selection_algorithm; 
00106         char *output;                                          
00107         struct list *input_files;                              
00108         struct list *output_files;                             
00109         struct list *env_list;                                 
00110         int taskid;                                            
00111         int return_status;                                     
00112         work_queue_result_t result;                       
00113         char *host;                                            
00114         char *hostname;                                        
00116         timestamp_t time_committed;                            
00118         timestamp_t time_task_submit;                          
00119         timestamp_t time_task_finish;                          
00120         timestamp_t time_send_input_start;                     
00121         timestamp_t time_send_input_finish;                    
00122         timestamp_t time_execute_cmd_start;                    
00123         timestamp_t time_execute_cmd_finish;                   
00124         timestamp_t time_receive_result_start;                 
00125         timestamp_t time_receive_result_finish;                
00126         timestamp_t time_receive_output_start;                 
00127         timestamp_t time_receive_output_finish;                
00129         int64_t total_bytes_received;                          
00130         int64_t total_bytes_sent;                              
00131         int64_t total_bytes_transferred;                       
00132         timestamp_t total_transfer_time;                       
00133         timestamp_t cmd_execution_time;                        
00134         int total_submissions;                                 
00135         timestamp_t total_cmd_execution_time;                  
00137         double priority;                                       
00139         int max_retries;                                       
00141         struct rmsummary *resources_measured;                  
00142         struct rmsummary *resources_requested;                 
00144         category_allocation_t resource_request;                
00146         char *category;                                        
00148         timestamp_t time_app_delay;                            
00149 };
00150 
00153 struct work_queue_stats {
00154         int total_workers_connected;    
00155         int workers_init;               
00156         int workers_idle;               
00157         int workers_busy;               
00159         int total_workers_joined;       
00160         int total_workers_removed;      
00161         int total_workers_lost;         
00162         int total_workers_idled_out;    
00163         int total_workers_fast_aborted; 
00165         /* Stats for the current state of tasks: */
00166         int tasks_waiting;              
00167         int tasks_running;              
00168         int tasks_complete;             
00170         /* Cummulative stats for tasks: */
00171         int total_tasks_dispatched;     
00172         int total_tasks_complete;       
00173         int total_tasks_failed;         
00174         int total_tasks_cancelled;      
00176         timestamp_t start_time;         
00177         timestamp_t total_send_time;    
00178         timestamp_t total_receive_time; 
00179         timestamp_t total_good_transfer_time;    
00181         timestamp_t total_execute_time;      
00182         timestamp_t total_good_execute_time; 
00185         int64_t total_bytes_sent;       
00186         int64_t total_bytes_received;   
00187         double efficiency;              
00188         double idle_percentage;         
00189         int capacity;                   
00191         double  bandwidth;              
00192         int64_t total_cores;            
00193         int64_t total_memory;           
00194         int64_t total_disk;                 
00195         int64_t total_gpus;             
00196         int64_t committed_cores;        
00197         int64_t committed_memory;       
00198         int64_t committed_disk;         
00199         int64_t committed_gpus;         
00200         int64_t min_cores;              
00201         int64_t max_cores;              
00202         int64_t min_memory;             
00203         int64_t max_memory;             
00204         int64_t min_disk;               
00205         int64_t max_disk;               
00206         int64_t min_gpus;               
00207         int64_t max_gpus;               
00208         int port;
00209         int priority;
00210         int workers_ready;              
00211         int workers_full;               
00212         int total_worker_slots;         
00213         int avg_capacity;               
00214 };
00215 
00216 
00220 
00228 struct work_queue_task *work_queue_task_create(const char *full_command);
00229 
00235 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00236 
00241 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00242 
00259 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t flags);
00260 
00275 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, work_queue_file_type_t type, work_queue_file_flags_t flags);
00276 
00287 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, work_queue_file_flags_t);
00288 
00302 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, work_queue_file_type_t type, work_queue_file_flags_t, int recursive);
00303 
00309 void work_queue_task_specify_max_retries( struct work_queue_task *t, int64_t max_retries );
00310 
00316 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00317 
00323 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00324 
00330 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00331 
00337 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00338 
00346 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t useconds );
00347 
00355 void work_queue_task_specify_running_time( struct work_queue_task *t, int64_t useconds );
00356 
00363 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00364 
00371 void work_queue_task_specify_category(struct work_queue_task *t, const char *category);
00372 
00379 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00380 
00387 void work_queue_task_specify_enviroment_variable( struct work_queue_task *t, const char *name, const char *value );
00388 
00394 void work_queue_task_specify_algorithm(struct work_queue_task *t, work_queue_schedule_t algorithm);
00395 
00400 void work_queue_task_delete(struct work_queue_task *t);
00401 
00403 
00407 
00424 struct work_queue *work_queue_create(int port);
00425 
00434 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00435 
00436 
00443 int work_queue_enable_monitoring_full(struct work_queue *q, char *monitor_output_directory);
00444 
00453 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00454 
00459 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00460 
00468 void work_queue_blacklist_add_with_timeout(struct work_queue *q, const char *hostname, time_t seconds);
00469 
00470 
00475 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00476 
00477 
00481 void work_queue_blacklist_clear(struct work_queue *q);
00482 
00496 void work_queue_invalidate_cached_file(struct work_queue *q, const char *local_name, work_queue_file_t type);
00497 
00498 
00513 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00514 
00526 int work_queue_hungry(struct work_queue *q);
00527 
00535 int work_queue_empty(struct work_queue *q);
00536 
00543 int work_queue_port(struct work_queue *q);
00544 
00549 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00550 
00555 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00556 
00562 void work_queue_get_stats_category(struct work_queue *q, const char *c, struct work_queue_stats *s);
00563 
00564 
00570 work_queue_task_state_t work_queue_task_state(struct work_queue *q, int taskid);
00571 
00576 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00577 
00582 double work_queue_get_effective_bandwidth(struct work_queue *q);
00583 
00590 char * work_queue_get_worker_summary( struct work_queue *q );
00591 
00601 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00602 
00603 
00613 int work_queue_activate_fast_abort_category(struct work_queue *q, const char *category, double multiplier);
00614 
00615 
00619 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00620 
00626 void work_queue_specify_algorithm(struct work_queue *q, work_queue_schedule_t algorithm);
00627 
00632 const char *work_queue_name(struct work_queue *q);
00633 
00638 void work_queue_specify_name(struct work_queue *q, const char *name);
00639 
00644 void work_queue_specify_priority(struct work_queue *q, int priority);
00645 
00654 void work_queue_specify_num_tasks_left(struct work_queue *q, int ntasks);
00655 
00661 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00662 
00668 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00669 
00675 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00676 
00681 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00682 
00687 int work_queue_shut_down_workers(struct work_queue *q, int n);
00688 
00693 void work_queue_delete(struct work_queue *q);
00694 
00700 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00701 
00707 void work_queue_specify_password( struct work_queue *q, const char *password );
00708 
00715 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00716 
00721 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00722 
00727 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00728 
00734 void work_queue_master_preferred_connection(struct work_queue *q, const char *preferred_connection);
00735 
00751 int work_queue_tune(struct work_queue *q, const char *name, double value);
00752 
00759 void work_queue_specify_max_resources(struct work_queue *q,  const struct rmsummary *rm);
00760 
00768 void work_queue_specify_max_category_resources(struct work_queue *q, const char *category, const struct rmsummary *rm);
00769 
00775 void work_queue_initialize_categories(struct work_queue *q, struct rmsummary *max, const char *summaries_file);
00776 
00777 
00779 
00783 
00784 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00785 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00793 void work_queue_specify_task_order(struct work_queue *q, int order);
00794 
00795 
00796 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00797 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00806 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00807 
00808 
00814 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00815 
00824 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00825 
00833 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00834 
00842 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00843 
00851 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00852 
00860 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00861 
00863 
00864 /* Experimental feature - intentionally left undocumented.
00865 This feature exists to simplify performance evaulation and is not recommended
00866 for production use since it delays execution of the workload.
00867 Force the master to wait for the given number of workers to connect before
00868 starting to dispatch tasks.
00869 @param q A work queue object.
00870 @param worker The number of workers to wait before tasks are dispatched.*/
00871 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00872 
00873 #endif

Generated on 14 Apr 2016 for cctools by  doxygen 1.6.1