work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 
00023 #define WORK_QUEUE_DEFAULT_PORT 9123  
00024 #define WORK_QUEUE_RANDOM_PORT  0    
00025 #define WORK_QUEUE_WAITFORTASK  -1    
00027 #define WORK_QUEUE_SCHEDULE_UNSET 0
00028 #define WORK_QUEUE_SCHEDULE_FCFS         1 
00029 #define WORK_QUEUE_SCHEDULE_FILES        2 
00030 #define WORK_QUEUE_SCHEDULE_TIME         3 
00031 #define WORK_QUEUE_SCHEDULE_RAND         4 
00033 #define WORK_QUEUE_INPUT  0     
00034 #define WORK_QUEUE_OUTPUT 1     
00036 #define WORK_QUEUE_NOCACHE 0    
00037 #define WORK_QUEUE_CACHE 1      
00038 #define WORK_QUEUE_SYMLINK 2    /* Create a symlink to the file rather than copying it, if possible. */
00039 #define WORK_QUEUE_PREEXIST 4   /* If the filename already exists on the host, use it in place. */
00040 #define WORK_QUEUE_THIRDGET 8   /* Access the file on the client from a shared filesystem */
00041 #define WORK_QUEUE_THIRDPUT 8   /* Access the file on the client from a shared filesystem (included for readability) */
00042 #define WORK_QUEUE_WATCH 16     
00044 #define WORK_QUEUE_RESET_ALL        0  
00045 #define WORK_QUEUE_RESET_KEEP_TASKS 1  
00047 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00048 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30    
00050 #define WORK_QUEUE_RESULT_SUCCESS 0                
00051 #define WORK_QUEUE_RESULT_INPUT_MISSING 1  
00052 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 
00053 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4 
00054 #define WORK_QUEUE_RESULT_SIGNAL         8 
00055 #define WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 
00056 #define WORK_QUEUE_RESULT_TASK_TIMEOUT 32 
00058 extern double wq_option_fast_abort_multiplier; 
00060 extern int wq_option_scheduler; 
00064 struct work_queue_task {
00065         char *tag;                      
00066         char *command_line;             
00067         int worker_selection_algorithm;           
00068         char *output;                   
00069         struct list *input_files;       
00070         struct list *output_files;      
00071         int taskid;                     
00072         int return_status;              
00073         int result;                     
00074         char *host;                     
00075         char *hostname;                 
00077         timestamp_t time_committed;     
00079         timestamp_t time_task_submit;   
00080         timestamp_t time_task_finish;   
00081         timestamp_t time_send_input_start;      
00082         timestamp_t time_send_input_finish;     
00083         timestamp_t time_execute_cmd_start;                 
00084         timestamp_t time_execute_cmd_finish;            
00085         timestamp_t time_receive_result_start;  
00086         timestamp_t time_receive_result_finish; 
00087         timestamp_t time_receive_output_start;  
00088         timestamp_t time_receive_output_finish; 
00090         int64_t total_bytes_received;
00091         int64_t total_bytes_sent;
00092         int64_t total_bytes_transferred;
00093         timestamp_t total_transfer_time;    
00094         timestamp_t cmd_execution_time;    
00095         int total_submissions;                     
00096         timestamp_t total_cmd_execution_time;   
00098         int64_t maximum_end_time;                       
00099         int64_t memory;                       
00100         int64_t disk;
00101         int cores;
00102         int gpus;
00103         int unlabeled;
00104         
00105         timestamp_t time_app_delay;      
00106         double priority;                        
00107 };
00108 
00111 struct work_queue_stats {
00112         int total_workers_connected;    
00113         int workers_init;               
00114         int workers_idle;               
00115         int workers_busy;               
00116         int total_workers_joined;       
00117         int total_workers_removed;      
00119         int tasks_waiting;              
00120         int tasks_running;              
00121         int tasks_complete;             
00122         int total_tasks_dispatched;     
00123         int total_tasks_complete;       
00124         int total_tasks_failed;         
00125         int total_tasks_cancelled;      
00127         timestamp_t start_time;         
00128         timestamp_t total_send_time;    
00129         timestamp_t total_receive_time; 
00130         timestamp_t total_good_transfer_time;    
00132         timestamp_t total_execute_time; 
00133         timestamp_t total_good_execute_time; 
00136         int64_t total_bytes_sent;       
00137         int64_t total_bytes_received;   
00138         double efficiency;              
00139         double idle_percentage;         
00140         int capacity;                   
00142         double  bandwidth;              
00143         int64_t total_cores;            
00144         int64_t total_memory;           
00145         int64_t total_disk;                 
00146         int64_t total_gpus;             
00147         int64_t committed_cores;        
00148         int64_t committed_memory;       
00149         int64_t committed_disk;         
00150         int64_t committed_gpus;         
00151         int64_t min_cores;              
00152         int64_t max_cores;              
00153         int64_t min_memory;             
00154         int64_t max_memory;             
00155         int64_t min_disk;               
00156         int64_t max_disk;               
00157         int64_t min_gpus;               
00158         int64_t max_gpus;               
00159         int port;                                               
00160         int priority;                                   
00161         int workers_ready;              
00162         int workers_full;               
00163         int total_worker_slots;         
00164         int avg_capacity;               
00165 };
00166 
00167 
00171 
00179 struct work_queue_task *work_queue_task_create(const char *full_command);
00180 
00186 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00187 
00192 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00193 
00210 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00211 
00226 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00227 
00238 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00239 
00253 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00254 
00260 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00261 
00267 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00268 
00274 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00275 
00281 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00282 
00288 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00289 
00296 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00297 
00304 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00305 
00315 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00316 
00321 void work_queue_task_delete(struct work_queue_task *t);
00322 
00324 
00328 
00345 struct work_queue *work_queue_create(int port);
00346 
00354 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00355 
00364 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00365 
00370 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00371 
00372 
00377 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00378 
00379 
00383 void work_queue_blacklist_clear(struct work_queue *q);
00384 
00399 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00400 
00412 int work_queue_hungry(struct work_queue *q);
00413 
00421 int work_queue_empty(struct work_queue *q);
00422 
00429 int work_queue_port(struct work_queue *q);
00430 
00435 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00436 
00441 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00442 
00447 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00448 
00453 double work_queue_get_effective_bandwidth(struct work_queue *q);
00454 
00461 char * work_queue_get_worker_summary( struct work_queue *q );
00462 
00468 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00469 
00470 
00474 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00475 
00485 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00486 
00491 const char *work_queue_name(struct work_queue *q);
00492 
00497 void work_queue_specify_name(struct work_queue *q, const char *name);
00498 
00503 void work_queue_specify_priority(struct work_queue *q, int priority);
00504 
00510 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00511 
00517 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00518 
00524 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00525 
00530 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00531 
00536 int work_queue_shut_down_workers(struct work_queue *q, int n);
00537 
00542 void work_queue_delete(struct work_queue *q);
00543 
00549 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00550 
00556 void work_queue_specify_password( struct work_queue *q, const char *password );
00557 
00564 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00565 
00570 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00571 
00576 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00577 
00578 
00594 int work_queue_tune(struct work_queue *q, const char *name, double value);
00595 
00597 
00601 
00602 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00603 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00611 void work_queue_specify_task_order(struct work_queue *q, int order);
00612 
00613 
00614 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00615 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00624 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00625 
00631 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00632 
00641 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00642 
00650 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00651 
00659 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00660 
00668 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00669 
00677 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00678 
00680 
00681 /* Experimental feature - intentionally left undocumented.
00682 This feature exists to simplify performance evaulation and is not recommended
00683 for production use since it delays execution of the workload. 
00684 Force the master to wait for the given number of workers to connect before
00685 starting to dispatch tasks.  
00686 @param q A work queue object.
00687 @param worker The number of workers to wait before tasks are dispatched.*/
00688 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00689 
00690 #endif

Generated on 24 Sep 2014 for cctools by  doxygen 1.6.1