work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 
00023 #define WORK_QUEUE_DEFAULT_PORT 9123  
00024 #define WORK_QUEUE_RANDOM_PORT  0    
00025 #define WORK_QUEUE_WAITFORTASK  -1    
00027 #define WORK_QUEUE_SCHEDULE_UNSET 0
00028 #define WORK_QUEUE_SCHEDULE_FCFS         1 
00029 #define WORK_QUEUE_SCHEDULE_FILES        2 
00030 #define WORK_QUEUE_SCHEDULE_TIME         3 
00031 #define WORK_QUEUE_SCHEDULE_RAND         4 
00033 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00034 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00036 #define WORK_QUEUE_INPUT  0     
00037 #define WORK_QUEUE_OUTPUT 1     
00039 #define WORK_QUEUE_NOCACHE 0    
00040 #define WORK_QUEUE_CACHE 1      
00041 #define WORK_QUEUE_SYMLINK 2    /* Create a symlink to the file rather than copying it, if possible. */
00042 #define WORK_QUEUE_PREEXIST 4   /* If the filename already exists on the host, use it in place. */
00043 #define WORK_QUEUE_THIRDGET 8   /* Access the file on the client from a shared filesystem */
00044 #define WORK_QUEUE_THIRDPUT 8   /* Access the file on the client from a shared filesystem (included for readability) */
00045 
00046 #define WORK_QUEUE_RESET_ALL        0  
00047 #define WORK_QUEUE_RESET_KEEP_TASKS 1  
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00050 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30    
00052 extern double wq_option_fast_abort_multiplier; 
00053 extern int wq_option_scheduler; 
00057 struct work_queue_task {
00058         char *tag;                      
00059         char *command_line;             
00060         int worker_selection_algorithm;           
00061         char *output;                   
00062         struct list *input_files;       
00063         struct list *output_files;      
00064         int taskid;                     
00065         int return_status;              
00066         int result;                     
00067         char *host;                     
00068         char *hostname;                 
00070         timestamp_t time_task_submit;   
00071         timestamp_t time_task_finish;   
00072         timestamp_t time_send_input_start;      
00073         timestamp_t time_send_input_finish;     
00074         timestamp_t time_execute_cmd_start;                 
00075         timestamp_t time_execute_cmd_finish;            
00076         timestamp_t time_receive_result_start;  
00077         timestamp_t time_receive_result_finish; 
00078         timestamp_t time_receive_output_start;  
00079         timestamp_t time_receive_output_finish; 
00081         int64_t total_bytes_transferred;
00082         timestamp_t total_transfer_time;    
00083         timestamp_t cmd_execution_time;    
00085         int64_t memory;                       
00086         int64_t disk;
00087         int cores;
00088         int gpus;
00089         int unlabeled;
00090         
00091         timestamp_t time_app_delay;      
00092 };
00093 
00096 struct work_queue_stats {
00097         int total_workers_connected;    
00098         int workers_init;               
00099         int workers_idle;               
00100         int workers_busy;               
00101         int total_workers_joined;       
00102         int total_workers_removed;      
00104         int tasks_waiting;              
00105         int tasks_running;              
00106         int tasks_complete;             
00107         int total_tasks_dispatched;     
00108         int total_tasks_complete;       
00109         int total_tasks_cancelled;      
00111         timestamp_t start_time;         
00112         timestamp_t total_send_time;    
00113         timestamp_t total_receive_time; 
00114         int64_t total_bytes_sent;       
00115         int64_t total_bytes_received;   
00116         double efficiency;              
00117         double idle_percentage;         
00118         int capacity;                   
00120         double  bandwidth;              
00121         int64_t total_cores;            
00122         int64_t total_memory;           
00123         int64_t total_disk;                 
00124         int64_t total_gpus;             
00125         int64_t min_cores;              
00126         int64_t max_cores;              
00127         int64_t min_memory;             
00128         int64_t max_memory;             
00129         int64_t min_disk;               
00130         int64_t max_disk;               
00131         int64_t min_gpus;               
00132         int64_t max_gpus;               
00134         int port;                                               
00135         int priority;                                   
00136         int workers_ready;              
00137         int workers_full;               
00138         int total_worker_slots;         
00139         int avg_capacity;               
00140 };
00141 
00142 
00146 
00154 struct work_queue_task *work_queue_task_create(const char *full_command);
00155 
00160 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00161 
00174 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00175 
00190 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00191 
00202 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00203 
00217 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00218 
00224 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00225 
00231 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00232 
00238 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00239 
00245 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00246 
00253 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00254 
00264 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00265 
00270 void work_queue_task_delete(struct work_queue_task *t);
00271 
00273 
00277 
00294 struct work_queue *work_queue_create(int port);
00295 
00303 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00304 
00313 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00314 
00329 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00330 
00342 int work_queue_hungry(struct work_queue *q);
00343 
00351 int work_queue_empty(struct work_queue *q);
00352 
00359 int work_queue_port(struct work_queue *q);
00360 
00365 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00366 
00371 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00372 
00377 double work_queue_get_effective_bandwidth(struct work_queue *q);
00378 
00385 char * work_queue_get_worker_summary( struct work_queue *q );
00386 
00392 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00393 
00405 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00406 
00416 void work_queue_specify_task_order(struct work_queue *q, int order);
00417 
00422 const char *work_queue_name(struct work_queue *q);
00423 
00428 void work_queue_specify_name(struct work_queue *q, const char *name);
00429 
00434 void work_queue_specify_priority(struct work_queue *q, int priority);
00435 
00441 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00442 
00448 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00449 
00455 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00456 
00461 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00462 
00469 void work_queue_reset(struct work_queue *q, int flags);
00470 
00475 int work_queue_shut_down_workers(struct work_queue *q, int n);
00476 
00481 void work_queue_delete(struct work_queue *q);
00482 
00488 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00489 
00495 void work_queue_specify_password( struct work_queue *q, const char *password );
00496 
00503 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00504 
00509 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00510 
00515 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00516 
00517 
00533 int work_queue_tune(struct work_queue *q, const char *name, double value);
00534 
00536 
00540 
00541 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00542 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00551 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00552 
00558 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00559 
00568 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00569 
00577 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00578 
00586 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00587 
00595 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00596 
00604 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00605 
00607 
00608 /* Experimental feature - intentionally left undocumented.
00609 This feature exists to simplify performance evaulation and is not recommended
00610 for production use since it delays execution of the workload. 
00611 Force the master to wait for the given number of workers to connect before
00612 starting to dispatch tasks.  
00613 @param q A work queue object.
00614 @param worker The number of workers to wait before tasks are dispatched.*/
00615 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00616 
00617 #endif

Generated on 25 Mar 2014 for cctools by  doxygen 1.6.1