work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 
00023 #define WORK_QUEUE_DEFAULT_PORT 9123  
00024 #define WORK_QUEUE_RANDOM_PORT  0    
00025 #define WORK_QUEUE_WAITFORTASK  -1    
00027 #define WORK_QUEUE_SCHEDULE_UNSET 0
00028 #define WORK_QUEUE_SCHEDULE_FCFS         1 
00029 #define WORK_QUEUE_SCHEDULE_FILES        2 
00030 #define WORK_QUEUE_SCHEDULE_TIME         3 
00031 #define WORK_QUEUE_SCHEDULE_RAND         4 
00033 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00034 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00036 #define WORK_QUEUE_INPUT  0     
00037 #define WORK_QUEUE_OUTPUT 1     
00039 #define WORK_QUEUE_NOCACHE 0    
00040 #define WORK_QUEUE_CACHE 1      
00041 #define WORK_QUEUE_SYMLINK 2    /* Create a symlink to the file rather than copying it, if possible. */
00042 #define WORK_QUEUE_PREEXIST 4   /* If the filename already exists on the host, use it in place. */
00043 #define WORK_QUEUE_THIRDGET 8   /* Access the file on the client from a shared filesystem */
00044 #define WORK_QUEUE_THIRDPUT 8   /* Access the file on the client from a shared filesystem (included for readability) */
00045 
00046 #define WORK_QUEUE_RESET_ALL        0  
00047 #define WORK_QUEUE_RESET_KEEP_TASKS 1  
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00050 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30    
00052 #define WORK_QUEUE_RESULT_SUCCESS 0                
00053 #define WORK_QUEUE_RESULT_INPUT_MISSING 1  
00054 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 
00055 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4 
00057 extern double wq_option_fast_abort_multiplier; 
00058 extern int wq_option_scheduler; 
00062 struct work_queue_task {
00063         char *tag;                      
00064         char *command_line;             
00065         int worker_selection_algorithm;           
00066         char *output;                   
00067         struct list *input_files;       
00068         struct list *output_files;      
00069         int taskid;                     
00070         int return_status;              
00071         int result;                     
00072         char *host;                     
00073         char *hostname;                 
00075         timestamp_t time_committed;     
00077         timestamp_t time_task_submit;   
00078         timestamp_t time_task_finish;   
00079         timestamp_t time_send_input_start;      
00080         timestamp_t time_send_input_finish;     
00081         timestamp_t time_execute_cmd_start;                 
00082         timestamp_t time_execute_cmd_finish;            
00083         timestamp_t time_receive_result_start;  
00084         timestamp_t time_receive_result_finish; 
00085         timestamp_t time_receive_output_start;  
00086         timestamp_t time_receive_output_finish; 
00088         int64_t total_bytes_received;
00089         int64_t total_bytes_sent;
00090         int64_t total_bytes_transferred;
00091         timestamp_t total_transfer_time;    
00092         timestamp_t cmd_execution_time;    
00093         int total_submissions;                     
00094         timestamp_t total_cmd_execution_time;   
00096         int64_t memory;                       
00097         int64_t disk;
00098         int cores;
00099         int gpus;
00100         int unlabeled;
00101         
00102         timestamp_t time_app_delay;      
00103 };
00104 
00107 struct work_queue_stats {
00108         int total_workers_connected;    
00109         int workers_init;               
00110         int workers_idle;               
00111         int workers_busy;               
00112         int total_workers_joined;       
00113         int total_workers_removed;      
00115         int tasks_waiting;              
00116         int tasks_running;              
00117         int tasks_complete;             
00118         int total_tasks_dispatched;     
00119         int total_tasks_complete;       
00120         int total_tasks_cancelled;      
00122         timestamp_t start_time;         
00123         timestamp_t total_send_time;    
00124         timestamp_t total_receive_time; 
00125         int64_t total_bytes_sent;       
00126         int64_t total_bytes_received;   
00127         double efficiency;              
00128         double idle_percentage;         
00129         int capacity;                   
00131         double  bandwidth;              
00132         int64_t total_cores;            
00133         int64_t total_memory;           
00134         int64_t total_disk;                 
00135         int64_t total_gpus;             
00136         int64_t committed_cores;        
00137         int64_t committed_memory;       
00138         int64_t committed_disk;         
00139         int64_t committed_gpus;         
00140         int64_t min_cores;              
00141         int64_t max_cores;              
00142         int64_t min_memory;             
00143         int64_t max_memory;             
00144         int64_t min_disk;               
00145         int64_t max_disk;               
00146         int64_t min_gpus;               
00147         int64_t max_gpus;               
00148         timestamp_t total_execute_time; 
00149         timestamp_t total_good_execute_time; 
00151         int port;                                               
00152         int priority;                                   
00153         int workers_ready;              
00154         int workers_full;               
00155         int total_worker_slots;         
00156         int avg_capacity;               
00157 };
00158 
00159 
00163 
00171 struct work_queue_task *work_queue_task_create(const char *full_command);
00172 
00178 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00179 
00184 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00185 
00198 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00199 
00214 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00215 
00226 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00227 
00241 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00242 
00248 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00249 
00255 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00256 
00262 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00263 
00269 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00270 
00277 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00278 
00288 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00289 
00294 void work_queue_task_delete(struct work_queue_task *t);
00295 
00297 
00301 
00318 struct work_queue *work_queue_create(int port);
00319 
00327 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00328 
00337 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00338 
00353 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00354 
00366 int work_queue_hungry(struct work_queue *q);
00367 
00375 int work_queue_empty(struct work_queue *q);
00376 
00383 int work_queue_port(struct work_queue *q);
00384 
00389 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00390 
00395 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00396 
00401 double work_queue_get_effective_bandwidth(struct work_queue *q);
00402 
00409 char * work_queue_get_worker_summary( struct work_queue *q );
00410 
00416 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00417 
00429 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00430 
00440 void work_queue_specify_task_order(struct work_queue *q, int order);
00441 
00446 const char *work_queue_name(struct work_queue *q);
00447 
00452 void work_queue_specify_name(struct work_queue *q, const char *name);
00453 
00458 void work_queue_specify_priority(struct work_queue *q, int priority);
00459 
00465 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00466 
00472 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00473 
00479 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00480 
00485 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00486 
00493 void work_queue_reset(struct work_queue *q, int flags);
00494 
00499 int work_queue_shut_down_workers(struct work_queue *q, int n);
00500 
00505 void work_queue_delete(struct work_queue *q);
00506 
00512 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00513 
00519 void work_queue_specify_password( struct work_queue *q, const char *password );
00520 
00527 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00528 
00533 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00534 
00539 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00540 
00541 
00557 int work_queue_tune(struct work_queue *q, const char *name, double value);
00558 
00560 
00564 
00565 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00566 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00575 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00576 
00582 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00583 
00592 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00593 
00601 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00602 
00610 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00611 
00619 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00620 
00628 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00629 
00631 
00632 /* Experimental feature - intentionally left undocumented.
00633 This feature exists to simplify performance evaulation and is not recommended
00634 for production use since it delays execution of the workload. 
00635 Force the master to wait for the given number of workers to connect before
00636 starting to dispatch tasks.  
00637 @param q A work queue object.
00638 @param worker The number of workers to wait before tasks are dispatched.*/
00639 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00640 
00641 #endif

Generated on 15 Aug 2014 for cctools by  doxygen 1.6.1