work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 #include "rmsummary.h"
00023 
00024 #define WORK_QUEUE_DEFAULT_PORT 9123  
00025 #define WORK_QUEUE_RANDOM_PORT  0    
00026 #define WORK_QUEUE_WAITFORTASK  -1    
00028 #define WORK_QUEUE_SCHEDULE_UNSET 0
00029 #define WORK_QUEUE_SCHEDULE_FCFS         1 
00030 #define WORK_QUEUE_SCHEDULE_FILES        2 
00031 #define WORK_QUEUE_SCHEDULE_TIME         3 
00032 #define WORK_QUEUE_SCHEDULE_RAND         4 
00033 #define WORK_QUEUE_SCHEDULE_WORST        5 
00035 #define WORK_QUEUE_INPUT  0     
00036 #define WORK_QUEUE_OUTPUT 1     
00038 #define WORK_QUEUE_NOCACHE 0    
00039 #define WORK_QUEUE_CACHE 1      
00040 #define WORK_QUEUE_SYMLINK 2    /* Create a symlink to the file rather than copying it, if possible. */
00041 #define WORK_QUEUE_PREEXIST 4   /* If the filename already exists on the host, use it in place. */
00042 #define WORK_QUEUE_THIRDGET 8   /* Access the file on the client from a shared filesystem */
00043 #define WORK_QUEUE_THIRDPUT 8   /* Access the file on the client from a shared filesystem (included for readability) */
00044 #define WORK_QUEUE_WATCH 16     
00046 #define WORK_QUEUE_RESET_ALL        0  
00047 #define WORK_QUEUE_RESET_KEEP_TASKS 1  
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00050 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30    
00052 #define WORK_QUEUE_RESULT_SUCCESS 0                
00053 #define WORK_QUEUE_RESULT_INPUT_MISSING 1  
00054 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 
00055 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4 
00056 #define WORK_QUEUE_RESULT_SIGNAL         8 
00057 #define WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 
00058 #define WORK_QUEUE_RESULT_TASK_TIMEOUT 32 
00061 #define WORK_QUEUE_TASK_UNKNOWN           0  
00062 #define WORK_QUEUE_TASK_READY             1  
00063 #define WORK_QUEUE_TASK_RUNNING           2  
00064 #define WORK_QUEUE_TASK_WAITING_RETRIEVAL 3  
00065 #define WORK_QUEUE_TASK_RETRIEVED         4  
00066 #define WORK_QUEUE_TASK_DONE              5  
00067 #define WORK_QUEUE_TASK_CANCELED          6  
00069 extern double wq_option_fast_abort_multiplier; 
00071 extern int wq_option_scheduler; 
00075 struct work_queue_task {
00076         char *tag;                      
00077         char *command_line;             
00078         int worker_selection_algorithm;           
00079         char *output;                   
00080         struct list *input_files;       
00081         struct list *output_files;      
00082         struct list *env_list;          
00083         int taskid;                     
00084         int return_status;              
00085         int result;                     
00086         char *host;                     
00087         char *hostname;                 
00089         timestamp_t time_committed;     
00091         timestamp_t time_task_submit;   
00092         timestamp_t time_task_finish;   
00093         timestamp_t time_send_input_start;      
00094         timestamp_t time_send_input_finish;     
00095         timestamp_t time_execute_cmd_start;                 
00096         timestamp_t time_execute_cmd_finish;            
00097         timestamp_t time_receive_result_start;  
00098         timestamp_t time_receive_result_finish; 
00099         timestamp_t time_receive_output_start;  
00100         timestamp_t time_receive_output_finish; 
00102         int64_t total_bytes_received;
00103         int64_t total_bytes_sent;
00104         int64_t total_bytes_transferred;
00105         timestamp_t total_transfer_time;    
00106         timestamp_t cmd_execution_time;    
00107         int total_submissions;                     
00108         timestamp_t total_cmd_execution_time;   
00110         int64_t maximum_end_time;
00111         int64_t memory;
00112         int64_t disk;
00113         int cores;
00114         int gpus;
00115         int unlabeled;
00116 
00117         double priority;                        
00119         struct rmsummary *resources_measured;   
00121         timestamp_t time_app_delay;      
00123 };
00124 
00127 struct work_queue_stats {
00128         int total_workers_connected;    
00129         int workers_init;               
00130         int workers_idle;               
00131         int workers_busy;               
00132         int total_workers_joined;       
00133         int total_workers_removed;      
00135         int tasks_waiting;              
00136         int tasks_running;              
00137         int tasks_complete;             
00138         int total_tasks_dispatched;     
00139         int total_tasks_complete;       
00140         int total_tasks_failed;         
00141         int total_tasks_cancelled;      
00143         timestamp_t start_time;         
00144         timestamp_t total_send_time;    
00145         timestamp_t total_receive_time; 
00146         timestamp_t total_good_transfer_time;    
00148         timestamp_t total_execute_time; 
00149         timestamp_t total_good_execute_time; 
00152         int64_t total_bytes_sent;       
00153         int64_t total_bytes_received;   
00154         double efficiency;              
00155         double idle_percentage;         
00156         int capacity;                   
00158         double  bandwidth;              
00159         int64_t total_cores;            
00160         int64_t total_memory;           
00161         int64_t total_disk;                 
00162         int64_t total_gpus;             
00163         int64_t committed_cores;        
00164         int64_t committed_memory;       
00165         int64_t committed_disk;         
00166         int64_t committed_gpus;         
00167         int64_t min_cores;              
00168         int64_t max_cores;              
00169         int64_t min_memory;             
00170         int64_t max_memory;             
00171         int64_t min_disk;               
00172         int64_t max_disk;               
00173         int64_t min_gpus;               
00174         int64_t max_gpus;               
00175         int port;
00176         int priority;
00177         int workers_ready;              
00178         int workers_full;               
00179         int total_worker_slots;         
00180         int avg_capacity;               
00181 };
00182 
00183 
00187 
00195 struct work_queue_task *work_queue_task_create(const char *full_command);
00196 
00202 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00203 
00208 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00209 
00226 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00227 
00242 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00243 
00254 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00255 
00269 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00270 
00276 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00277 
00283 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00284 
00290 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00291 
00297 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00298 
00304 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00305 
00312 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00313 
00320 void work_queue_task_specify_priority(struct work_queue_task *t, double priority );
00321 
00328 void work_queue_task_specify_env( struct work_queue_task *t, const char *name, const char *value );
00329 
00339 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00340 
00345 void work_queue_task_delete(struct work_queue_task *t);
00346 
00348 
00352 
00369 struct work_queue *work_queue_create(int port);
00370 
00378 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00379 
00388 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00389 
00394 void work_queue_blacklist_add(struct work_queue *q, const char *hostname);
00395 
00396 
00401 void work_queue_blacklist_remove(struct work_queue *q, const char *hostname);
00402 
00403 
00407 void work_queue_blacklist_clear(struct work_queue *q);
00408 
00423 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00424 
00436 int work_queue_hungry(struct work_queue *q);
00437 
00445 int work_queue_empty(struct work_queue *q);
00446 
00453 int work_queue_port(struct work_queue *q);
00454 
00459 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00460 
00465 void work_queue_get_stats_hierarchy(struct work_queue *q, struct work_queue_stats *s);
00466 
00467 
00473 int work_queue_task_state(struct work_queue *q, int taskid);
00474 
00479 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00480 
00485 double work_queue_get_effective_bandwidth(struct work_queue *q);
00486 
00493 char * work_queue_get_worker_summary( struct work_queue *q );
00494 
00500 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00501 
00502 
00506 int work_queue_send_receive_ratio(struct work_queue *q, double ratio);
00507 
00517 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00518 
00523 const char *work_queue_name(struct work_queue *q);
00524 
00529 void work_queue_specify_name(struct work_queue *q, const char *name);
00530 
00535 void work_queue_specify_priority(struct work_queue *q, int priority);
00536 
00542 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00543 
00549 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00550 
00556 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00557 
00562 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00563 
00568 int work_queue_shut_down_workers(struct work_queue *q, int n);
00569 
00574 void work_queue_delete(struct work_queue *q);
00575 
00581 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00582 
00588 void work_queue_specify_password( struct work_queue *q, const char *password );
00589 
00596 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00597 
00602 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00603 
00608 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00609 
00610 
00626 int work_queue_tune(struct work_queue *q, const char *name, double value);
00627 
00629 
00633 
00634 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00635 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00643 void work_queue_specify_task_order(struct work_queue *q, int order);
00644 
00645 
00646 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00647 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00656 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00657 
00663 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00664 
00673 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00674 
00682 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00683 
00691 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00692 
00700 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00701 
00709 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00710 
00712 
00713 /* Experimental feature - intentionally left undocumented.
00714 This feature exists to simplify performance evaulation and is not recommended
00715 for production use since it delays execution of the workload.
00716 Force the master to wait for the given number of workers to connect before
00717 starting to dispatch tasks.
00718 @param q A work queue object.
00719 @param worker The number of workers to wait before tasks are dispatched.*/
00720 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00721 
00722 #endif

Generated on 29 Jun 2015 for cctools by  doxygen 1.6.1