work_queue.h

Go to the documentation of this file.
00001 /*
00002 Copyright (C) 2008- The University of Notre Dame
00003 This software is distributed under the GNU General Public License.
00004 See the file COPYING for details.
00005 */
00006 
00007 #ifndef WORK_QUEUE_H
00008 #define WORK_QUEUE_H
00009 
00020 #include <sys/types.h>
00021 #include "timestamp.h"
00022 
00023 #define WORK_QUEUE_DEFAULT_PORT 9123  
00024 #define WORK_QUEUE_RANDOM_PORT  0    
00025 #define WORK_QUEUE_WAITFORTASK  -1    
00027 #define WORK_QUEUE_SCHEDULE_UNSET 0
00028 #define WORK_QUEUE_SCHEDULE_FCFS         1 
00029 #define WORK_QUEUE_SCHEDULE_FILES        2 
00030 #define WORK_QUEUE_SCHEDULE_TIME         3 
00031 #define WORK_QUEUE_SCHEDULE_RAND         4 
00033 #define WORK_QUEUE_TASK_ORDER_FIFO 0  
00034 #define WORK_QUEUE_TASK_ORDER_LIFO 1  
00036 #define WORK_QUEUE_INPUT  0     
00037 #define WORK_QUEUE_OUTPUT 1     
00039 #define WORK_QUEUE_NOCACHE 0    
00040 #define WORK_QUEUE_CACHE 1      
00041 #define WORK_QUEUE_SYMLINK 2    /* Create a symlink to the file rather than copying it, if possible. */
00042 #define WORK_QUEUE_PREEXIST 4   /* If the filename already exists on the host, use it in place. */
00043 #define WORK_QUEUE_THIRDGET 8   /* Access the file on the client from a shared filesystem */
00044 #define WORK_QUEUE_THIRDPUT 8   /* Access the file on the client from a shared filesystem (included for readability) */
00045 
00046 #define WORK_QUEUE_RESET_ALL        0  
00047 #define WORK_QUEUE_RESET_KEEP_TASKS 1  
00049 #define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300  
00050 #define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30    
00052 #define WORK_QUEUE_RESULT_SUCCESS 0                
00053 #define WORK_QUEUE_RESULT_INPUT_MISSING 1  
00054 #define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 
00055 #define WORK_QUEUE_RESULT_STDOUT_MISSING 4 
00056 #define WORK_QUEUE_RESULT_SIGNAL         8 
00057 #define WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 
00058 #define WORK_QUEUE_RESULT_TASK_TIMEOUT 32 
00060 extern double wq_option_fast_abort_multiplier; 
00062 extern int wq_option_scheduler; 
00066 struct work_queue_task {
00067         char *tag;                      
00068         char *command_line;             
00069         int worker_selection_algorithm;           
00070         char *output;                   
00071         struct list *input_files;       
00072         struct list *output_files;      
00073         int taskid;                     
00074         int return_status;              
00075         int result;                     
00076         char *host;                     
00077         char *hostname;                 
00079         timestamp_t time_committed;     
00081         timestamp_t time_task_submit;   
00082         timestamp_t time_task_finish;   
00083         timestamp_t time_send_input_start;      
00084         timestamp_t time_send_input_finish;     
00085         timestamp_t time_execute_cmd_start;                 
00086         timestamp_t time_execute_cmd_finish;            
00087         timestamp_t time_receive_result_start;  
00088         timestamp_t time_receive_result_finish; 
00089         timestamp_t time_receive_output_start;  
00090         timestamp_t time_receive_output_finish; 
00092         int64_t total_bytes_received;
00093         int64_t total_bytes_sent;
00094         int64_t total_bytes_transferred;
00095         timestamp_t total_transfer_time;    
00096         timestamp_t cmd_execution_time;    
00097         int total_submissions;                     
00098         timestamp_t total_cmd_execution_time;   
00100         int64_t maximum_end_time;                       
00101         int64_t memory;                       
00102         int64_t disk;
00103         int cores;
00104         int gpus;
00105         int unlabeled;
00106         
00107         timestamp_t time_app_delay;      
00108 };
00109 
00112 struct work_queue_stats {
00113         int total_workers_connected;    
00114         int workers_init;               
00115         int workers_idle;               
00116         int workers_busy;               
00117         int total_workers_joined;       
00118         int total_workers_removed;      
00120         int tasks_waiting;              
00121         int tasks_running;              
00122         int tasks_complete;             
00123         int total_tasks_dispatched;     
00124         int total_tasks_complete;       
00125         int total_tasks_cancelled;      
00127         timestamp_t start_time;         
00128         timestamp_t total_send_time;    
00129         timestamp_t total_receive_time; 
00130         int64_t total_bytes_sent;       
00131         int64_t total_bytes_received;   
00132         double efficiency;              
00133         double idle_percentage;         
00134         int capacity;                   
00136         double  bandwidth;              
00137         int64_t total_cores;            
00138         int64_t total_memory;           
00139         int64_t total_disk;                 
00140         int64_t total_gpus;             
00141         int64_t committed_cores;        
00142         int64_t committed_memory;       
00143         int64_t committed_disk;         
00144         int64_t committed_gpus;         
00145         int64_t min_cores;              
00146         int64_t max_cores;              
00147         int64_t min_memory;             
00148         int64_t max_memory;             
00149         int64_t min_disk;               
00150         int64_t max_disk;               
00151         int64_t min_gpus;               
00152         int64_t max_gpus;               
00153         timestamp_t total_execute_time; 
00154         timestamp_t total_good_execute_time; 
00156         int port;                                               
00157         int priority;                                   
00158         int workers_ready;              
00159         int workers_full;               
00160         int total_worker_slots;         
00161         int avg_capacity;               
00162 };
00163 
00164 
00168 
00176 struct work_queue_task *work_queue_task_create(const char *full_command);
00177 
00183 struct work_queue_task *work_queue_task_clone(const struct work_queue_task *task);
00184 
00189 void work_queue_task_specify_command( struct work_queue_task *t, const char *cmd );
00190 
00203 int work_queue_task_specify_file(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags);
00204 
00219 int work_queue_task_specify_file_piece(struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags);
00220 
00231 int work_queue_task_specify_buffer(struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags);
00232 
00246 int work_queue_task_specify_directory(struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive);
00247 
00253 void work_queue_task_specify_memory( struct work_queue_task *t, int64_t memory );
00254 
00260 void work_queue_task_specify_disk( struct work_queue_task *t, int64_t disk );
00261 
00267 void work_queue_task_specify_cores( struct work_queue_task *t, int cores );
00268 
00274 void work_queue_task_specify_gpus( struct work_queue_task *t, int gpus );
00275 
00281 void work_queue_task_specify_end_time( struct work_queue_task *t, int64_t seconds );
00282 
00289 void work_queue_task_specify_tag(struct work_queue_task *t, const char *tag);
00290 
00300 void work_queue_task_specify_algorithm(struct work_queue_task *t, int algo );
00301 
00306 void work_queue_task_delete(struct work_queue_task *t);
00307 
00309 
00313 
00330 struct work_queue *work_queue_create(int port);
00331 
00339 int work_queue_enable_monitoring(struct work_queue *q, char *monitor_summary_file);
00340 
00349 int work_queue_submit(struct work_queue *q, struct work_queue_task *t);
00350 
00365 struct work_queue_task *work_queue_wait(struct work_queue *q, int timeout);
00366 
00378 int work_queue_hungry(struct work_queue *q);
00379 
00387 int work_queue_empty(struct work_queue *q);
00388 
00395 int work_queue_port(struct work_queue *q);
00396 
00401 void work_queue_get_stats(struct work_queue *q, struct work_queue_stats *s);
00402 
00407 void work_queue_set_bandwidth_limit(struct work_queue *q, const char *bandwidth);
00408 
00413 double work_queue_get_effective_bandwidth(struct work_queue *q);
00414 
00421 char * work_queue_get_worker_summary( struct work_queue *q );
00422 
00428 int work_queue_activate_fast_abort(struct work_queue *q, double multiplier);
00429 
00430 
00442 void work_queue_specify_algorithm(struct work_queue *q, int algo);
00443 
00453 void work_queue_specify_task_order(struct work_queue *q, int order);
00454 
00459 const char *work_queue_name(struct work_queue *q);
00460 
00465 void work_queue_specify_name(struct work_queue *q, const char *name);
00466 
00471 void work_queue_specify_priority(struct work_queue *q, int priority);
00472 
00478 void work_queue_specify_catalog_server(struct work_queue *q, const char *hostname, int port);
00479 
00485 struct work_queue_task *work_queue_cancel_by_taskid(struct work_queue *q, int id);
00486 
00492 struct work_queue_task *work_queue_cancel_by_tasktag(struct work_queue *q, const char *tag);
00493 
00498 struct list * work_queue_cancel_all_tasks(struct work_queue *q);
00499 
00506 void work_queue_reset(struct work_queue *q, int flags);
00507 
00512 int work_queue_shut_down_workers(struct work_queue *q, int n);
00513 
00518 void work_queue_delete(struct work_queue *q);
00519 
00525 int work_queue_specify_log(struct work_queue *q, const char *logfile);
00526 
00532 void work_queue_specify_password( struct work_queue *q, const char *password );
00533 
00540 int work_queue_specify_password_file( struct work_queue *q, const char *file );
00541 
00546 void work_queue_specify_keepalive_interval(struct work_queue *q, int interval);
00547 
00552 void work_queue_specify_keepalive_timeout(struct work_queue *q, int timeout);
00553 
00554 
00570 int work_queue_tune(struct work_queue *q, const char *name, double value);
00571 
00573 
00577 
00578 #define WORK_QUEUE_MASTER_MODE_STANDALONE 0 
00579 #define WORK_QUEUE_MASTER_MODE_CATALOG 1    
00588 void work_queue_specify_master_mode(struct work_queue *q, int mode);
00589 
00595 void work_queue_specify_estimate_capacity_on(struct work_queue *q, int estimate_capacity_on);
00596 
00605 int work_queue_task_specify_input_buf(struct work_queue_task *t, const char *buf, int length, const char *rname);
00606 
00614 int work_queue_task_specify_input_file(struct work_queue_task *t, const char *fname, const char *rname);
00615 
00623 int work_queue_task_specify_input_file_do_not_cache(struct work_queue_task *t, const char *fname, const char *rname);
00624 
00632 int work_queue_task_specify_output_file(struct work_queue_task *t, const char *rname, const char *fname);
00633 
00641 int work_queue_task_specify_output_file_do_not_cache(struct work_queue_task *t, const char *rname, const char *fname);
00642 
00644 
00645 /* Experimental feature - intentionally left undocumented.
00646 This feature exists to simplify performance evaulation and is not recommended
00647 for production use since it delays execution of the workload. 
00648 Force the master to wait for the given number of workers to connect before
00649 starting to dispatch tasks.  
00650 @param q A work queue object.
00651 @param worker The number of workers to wait before tasks are dispatched.*/
00652 void work_queue_activate_worker_waiting(struct work_queue *q, int resources);
00653 
00654 #endif

Generated on 25 Aug 2014 for cctools by  doxygen 1.6.1