cctools
|
A master-worker library. More...
Go to the source code of this file.
Data Structures | |
struct | work_queue_task |
A task description. More... | |
struct | work_queue_stats |
Statistics describing a work queue. More... | |
Macros | |
#define | WORK_QUEUE_DEFAULT_PORT 9123 |
Default Work Queue port number. More... | |
#define | WORK_QUEUE_RANDOM_PORT 0 |
Indicates that any port number may be chosen. More... | |
#define | WORK_QUEUE_WAITFORTASK -1 |
Wait for a task to complete before returning. More... | |
#define | WORK_QUEUE_SCHEDULE_FCFS 1 |
Select worker on a first-come-first-serve basis. More... | |
#define | WORK_QUEUE_SCHEDULE_FILES 2 |
Select worker that has the most data required by the task. More... | |
#define | WORK_QUEUE_SCHEDULE_TIME 3 |
Select worker that has the fastest execution time on previous tasks. More... | |
#define | WORK_QUEUE_SCHEDULE_RAND 4 |
Select a random worker. More... | |
#define | WORK_QUEUE_INPUT 0 |
Specify an input object. More... | |
#define | WORK_QUEUE_OUTPUT 1 |
Specify an output object. More... | |
#define | WORK_QUEUE_NOCACHE 0 |
Do not cache file at execution site. More... | |
#define | WORK_QUEUE_CACHE 1 |
Cache file at execution site for later use. More... | |
#define | WORK_QUEUE_WATCH 16 |
Watch the output file and send back changes as the task runs. More... | |
#define | WORK_QUEUE_RESET_ALL 0 |
When resetting, clear out all tasks and files. More... | |
#define | WORK_QUEUE_RESET_KEEP_TASKS 1 |
When resetting, keep the current list of tasks. More... | |
#define | WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300 |
Default value for Work Queue keepalive interval in seconds. More... | |
#define | WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30 |
Default value for Work Queue keepalive timeout in seconds. More... | |
#define | WORK_QUEUE_RESULT_SUCCESS 0 |
The task ran successfully > More... | |
#define | WORK_QUEUE_RESULT_INPUT_MISSING 1 |
The task cannot be run due to a missing input file > More... | |
#define | WORK_QUEUE_RESULT_OUTPUT_MISSING 2 |
The task ran but failed to generate a specified output file > More... | |
#define | WORK_QUEUE_RESULT_STDOUT_MISSING 4 |
The task ran but its stdout has been truncated > More... | |
#define | WORK_QUEUE_RESULT_SIGNAL 8 |
The task was terminated with a signal > More... | |
#define | WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 |
The task used more resources than requested > More... | |
#define | WORK_QUEUE_RESULT_TASK_TIMEOUT 32 |
The task ran after specified end time. More... | |
#define | WORK_QUEUE_TASK_UNKNOWN 0 |
Task states. More... | |
#define | WORK_QUEUE_TASK_READY 1 |
Task is ready to be run, waiting in queue > More... | |
#define | WORK_QUEUE_TASK_RUNNING 2 |
Task has been dispatched to some worker > More... | |
#define | WORK_QUEUE_TASK_WAITING_RETRIEVAL 3 |
Task results are available at the worker > More... | |
#define | WORK_QUEUE_TASK_RETRIEVED 4 |
Task results are available at the master > More... | |
#define | WORK_QUEUE_TASK_DONE 5 |
Task is done, and has been returned through work_queue_wait > More... | |
#define | WORK_QUEUE_TASK_CANCELED 6 |
Task was canceled before completion > More... | |
Functions | |
Functions - Tasks | |
struct work_queue_task * | work_queue_task_create (const char *full_command) |
Create a new task object. More... | |
struct work_queue_task * | work_queue_task_clone (const struct work_queue_task *task) |
Create a copy of a task Create a functionally identical copy of a work_queue_task that can be re-submitted via work_queue_submit. More... | |
void | work_queue_task_specify_command (struct work_queue_task *t, const char *cmd) |
Indicate the command to be executed. More... | |
int | work_queue_task_specify_file (struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags) |
Add a file to a task. More... | |
int | work_queue_task_specify_file_piece (struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags) |
Add a file piece to a task. More... | |
int | work_queue_task_specify_buffer (struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags) |
Add an input buffer to a task. More... | |
int | work_queue_task_specify_directory (struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive) |
Add a directory to a task. More... | |
void | work_queue_task_specify_memory (struct work_queue_task *t, int64_t memory) |
Specify the amount of memory required by a task. More... | |
void | work_queue_task_specify_disk (struct work_queue_task *t, int64_t disk) |
Specify the amount of disk space required by a task. More... | |
void | work_queue_task_specify_cores (struct work_queue_task *t, int cores) |
Specify the number of cores required by a task. More... | |
void | work_queue_task_specify_gpus (struct work_queue_task *t, int gpus) |
Specify the number of gpus required by a task. More... | |
void | work_queue_task_specify_end_time (struct work_queue_task *t, int64_t seconds) |
Specify the maximum end time allowed for the task (in seconds since the Epoch). More... | |
void | work_queue_task_specify_tag (struct work_queue_task *t, const char *tag) |
Attach a user defined string tag to the task. More... | |
void | work_queue_task_specify_priority (struct work_queue_task *t, double priority) |
Specify the priority of this task relative to others in the queue. More... | |
void | work_queue_task_specify_algorithm (struct work_queue_task *t, int algo) |
Select the scheduling algorithm for a single task. More... | |
void | work_queue_task_delete (struct work_queue_task *t) |
Delete a task. More... | |
Functions - Queues | |
struct work_queue * | work_queue_create (int port) |
Create a new work queue. More... | |
int | work_queue_enable_monitoring (struct work_queue *q, char *monitor_summary_file) |
Enables resource monitoring on the give work queue. More... | |
int | work_queue_submit (struct work_queue *q, struct work_queue_task *t) |
Submit a task to a queue. More... | |
void | work_queue_blacklist_add (struct work_queue *q, const char *hostname) |
Blacklist host from a queue. More... | |
void | work_queue_blacklist_remove (struct work_queue *q, const char *hostname) |
Unblacklist host from a queue. More... | |
void | work_queue_blacklist_clear (struct work_queue *q) |
Clear blacklist of a queue. More... | |
struct work_queue_task * | work_queue_wait (struct work_queue *q, int timeout) |
Wait for a task to complete. More... | |
int | work_queue_hungry (struct work_queue *q) |
Determine whether the queue is 'hungry' for more tasks. More... | |
int | work_queue_empty (struct work_queue *q) |
Determine whether the queue is empty. More... | |
int | work_queue_port (struct work_queue *q) |
Get the listening port of the queue. More... | |
void | work_queue_get_stats (struct work_queue *q, struct work_queue_stats *s) |
Get queue statistics (only from master). More... | |
void | work_queue_get_stats_hierarchy (struct work_queue *q, struct work_queue_stats *s) |
Get statistics of the master queue together with foremen information. More... | |
int | work_queue_task_state (struct work_queue *q, int taskid) |
Get the current state of the task. More... | |
void | work_queue_set_bandwidth_limit (struct work_queue *q, const char *bandwidth) |
Limit the queue bandwidth when transferring files to and from workers. More... | |
double | work_queue_get_effective_bandwidth (struct work_queue *q) |
Get current queue bandwidth. More... | |
char * | work_queue_get_worker_summary (struct work_queue *q) |
Summarize workers. More... | |
int | work_queue_activate_fast_abort (struct work_queue *q, double multiplier) |
Turn on or off fast abort functionality for a given queue. More... | |
int | work_queue_send_receive_ratio (struct work_queue *q, double ratio) |
Change the preference to send or receive tasks. More... | |
void | work_queue_specify_algorithm (struct work_queue *q, int algo) |
Change the worker selection algorithm. More... | |
const char * | work_queue_name (struct work_queue *q) |
Get the project name of the queue. More... | |
void | work_queue_specify_name (struct work_queue *q, const char *name) |
Change the project name for a given queue. More... | |
void | work_queue_specify_priority (struct work_queue *q, int priority) |
Change the priority for a given queue. More... | |
void | work_queue_specify_catalog_server (struct work_queue *q, const char *hostname, int port) |
Specify the catalog server the master should report to. More... | |
struct work_queue_task * | work_queue_cancel_by_taskid (struct work_queue *q, int id) |
Cancel a submitted task using its task id and remove it from queue. More... | |
struct work_queue_task * | work_queue_cancel_by_tasktag (struct work_queue *q, const char *tag) |
Cancel a submitted task using its tag and remove it from queue. More... | |
struct list * | work_queue_cancel_all_tasks (struct work_queue *q) |
Cancel all submitted tasks and remove them from the queue. More... | |
int | work_queue_shut_down_workers (struct work_queue *q, int n) |
Shut down workers connected to the work_queue system. More... | |
void | work_queue_delete (struct work_queue *q) |
Delete a work queue. More... | |
int | work_queue_specify_log (struct work_queue *q, const char *logfile) |
Add a log file that records the states of the connected workers and submitted tasks. More... | |
void | work_queue_specify_password (struct work_queue *q, const char *password) |
Add a mandatory password that each worker must present. More... | |
int | work_queue_specify_password_file (struct work_queue *q, const char *file) |
Add a mandatory password file that each worker must present. More... | |
void | work_queue_specify_keepalive_interval (struct work_queue *q, int interval) |
Change the keepalive interval for a given queue. More... | |
void | work_queue_specify_keepalive_timeout (struct work_queue *q, int timeout) |
Change the keepalive timeout for identifying dead workers for a given queue. More... | |
int | work_queue_tune (struct work_queue *q, const char *name, double value) |
Tune advanced parameters for work queue. More... | |
Variables | |
double | wq_option_fast_abort_multiplier |
Initial setting for fast abort multiplier upon creating queue. More... | |
int | wq_option_scheduler |
Initial setting for algorithm to assign tasks to workers upon creating queue . More... | |
Functions - Deprecated | |
#define | WORK_QUEUE_TASK_ORDER_FIFO 0 |
Retrieve tasks based on first-in-first-out order. More... | |
#define | WORK_QUEUE_TASK_ORDER_LIFO 1 |
Retrieve tasks based on last-in-first-out order. More... | |
#define | WORK_QUEUE_MASTER_MODE_STANDALONE 0 |
Work Queue master does not report to the catalog server. More... | |
#define | WORK_QUEUE_MASTER_MODE_CATALOG 1 |
Work Queue master reports to catalog server. More... | |
void | work_queue_specify_task_order (struct work_queue *q, int order) |
Specified how the submitted tasks should be ordered. More... | |
void | work_queue_specify_master_mode (struct work_queue *q, int mode) |
Specify the master mode for a given queue. More... | |
void | work_queue_specify_estimate_capacity_on (struct work_queue *q, int estimate_capacity_on) |
Change whether to estimate master capacity for a given queue. More... | |
int | work_queue_task_specify_input_buf (struct work_queue_task *t, const char *buf, int length, const char *rname) |
Add an input buffer to a task. More... | |
int | work_queue_task_specify_input_file (struct work_queue_task *t, const char *fname, const char *rname) |
Add an input file to a task. More... | |
int | work_queue_task_specify_input_file_do_not_cache (struct work_queue_task *t, const char *fname, const char *rname) |
Add an input file to a task, without caching. More... | |
int | work_queue_task_specify_output_file (struct work_queue_task *t, const char *rname, const char *fname) |
Add an output file to a task. More... | |
int | work_queue_task_specify_output_file_do_not_cache (struct work_queue_task *t, const char *rname, const char *fname) |
Add an output file to a task without caching. More... | |
A master-worker library.
The work queue provides an implementation of the master-worker computing model using TCP sockets, Unix applications, and files as intermediate buffers. A master process uses work_queue_create to create a queue, then work_queue_submit to submit tasks. Once tasks are running, call work_queue_wait to wait for completion. A generic worker program, named work_queue_worker
, can be run on any machine, and simply needs to be told the host and port of the master.
#define WORK_QUEUE_DEFAULT_PORT 9123 |
Default Work Queue port number.
#define WORK_QUEUE_RANDOM_PORT 0 |
Indicates that any port number may be chosen.
#define WORK_QUEUE_WAITFORTASK -1 |
Wait for a task to complete before returning.
#define WORK_QUEUE_SCHEDULE_FCFS 1 |
Select worker on a first-come-first-serve basis.
#define WORK_QUEUE_SCHEDULE_FILES 2 |
Select worker that has the most data required by the task.
#define WORK_QUEUE_SCHEDULE_TIME 3 |
Select worker that has the fastest execution time on previous tasks.
#define WORK_QUEUE_SCHEDULE_RAND 4 |
Select a random worker.
(default)
#define WORK_QUEUE_INPUT 0 |
Specify an input object.
#define WORK_QUEUE_OUTPUT 1 |
Specify an output object.
#define WORK_QUEUE_NOCACHE 0 |
Do not cache file at execution site.
#define WORK_QUEUE_CACHE 1 |
Cache file at execution site for later use.
#define WORK_QUEUE_WATCH 16 |
Watch the output file and send back changes as the task runs.
#define WORK_QUEUE_RESET_ALL 0 |
When resetting, clear out all tasks and files.
#define WORK_QUEUE_RESET_KEEP_TASKS 1 |
When resetting, keep the current list of tasks.
#define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300 |
Default value for Work Queue keepalive interval in seconds.
#define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30 |
Default value for Work Queue keepalive timeout in seconds.
#define WORK_QUEUE_RESULT_SUCCESS 0 |
The task ran successfully >
#define WORK_QUEUE_RESULT_INPUT_MISSING 1 |
The task cannot be run due to a missing input file >
#define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 |
The task ran but failed to generate a specified output file >
#define WORK_QUEUE_RESULT_STDOUT_MISSING 4 |
The task ran but its stdout has been truncated >
#define WORK_QUEUE_RESULT_SIGNAL 8 |
The task was terminated with a signal >
#define WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 |
The task used more resources than requested >
#define WORK_QUEUE_RESULT_TASK_TIMEOUT 32 |
The task ran after specified end time.
>
#define WORK_QUEUE_TASK_UNKNOWN 0 |
Task states.
There is no such task >
#define WORK_QUEUE_TASK_READY 1 |
Task is ready to be run, waiting in queue >
#define WORK_QUEUE_TASK_RUNNING 2 |
Task has been dispatched to some worker >
#define WORK_QUEUE_TASK_WAITING_RETRIEVAL 3 |
Task results are available at the worker >
#define WORK_QUEUE_TASK_RETRIEVED 4 |
Task results are available at the master >
#define WORK_QUEUE_TASK_DONE 5 |
Task is done, and has been returned through work_queue_wait >
#define WORK_QUEUE_TASK_CANCELED 6 |
Task was canceled before completion >
#define WORK_QUEUE_TASK_ORDER_FIFO 0 |
Retrieve tasks based on first-in-first-out order.
#define WORK_QUEUE_TASK_ORDER_LIFO 1 |
Retrieve tasks based on last-in-first-out order.
#define WORK_QUEUE_MASTER_MODE_STANDALONE 0 |
Work Queue master does not report to the catalog server.
#define WORK_QUEUE_MASTER_MODE_CATALOG 1 |
Work Queue master reports to catalog server.
struct work_queue_task* work_queue_task_create | ( | const char * | full_command | ) |
Create a new task object.
Once created and elaborated with functions such as work_queue_task_specify_file and work_queue_task_specify_buffer, the task should be passed to work_queue_submit.
full_command | The shell command line to be executed by the task. If null, the command will be given later by work_queue_task_specify_command |
Referenced by work_queue.Task::__init__().
struct work_queue_task* work_queue_task_clone | ( | const struct work_queue_task * | task | ) |
Create a copy of a task
Create a functionally identical copy of a work_queue_task that can be re-submitted via work_queue_submit.
Referenced by work_queue.Task::clone().
void work_queue_task_specify_command | ( | struct work_queue_task * | t, |
const char * | cmd | ||
) |
Indicate the command to be executed.
t | A task object. |
cmd | The command to be executed. This string will be duplicated by this call, so the argument may be freed or re-used afterward. |
Referenced by work_queue.Task::specify_command().
int work_queue_task_specify_file | ( | struct work_queue_task * | t, |
const char * | local_name, | ||
const char * | remote_name, | ||
int | type, | ||
int | flags | ||
) |
Add a file to a task.
t | A task object. |
local_name | The name of the file on local disk or shared filesystem. |
remote_name | The name of the file at the remote execution site. |
type | Must be one of the following values:
|
flags | May be zero to indicate no special handling or any of the following or'd together:
|
Referenced by work_queue.Task::specify_file().
int work_queue_task_specify_file_piece | ( | struct work_queue_task * | t, |
const char * | local_name, | ||
const char * | remote_name, | ||
off_t | start_byte, | ||
off_t | end_byte, | ||
int | type, | ||
int | flags | ||
) |
Add a file piece to a task.
t | A task object. |
local_name | The name of the file on local disk or shared filesystem. |
remote_name | The name of the file at the remote execution site. |
start_byte | The starting byte offset of the file piece to be transferred. |
end_byte | The ending byte offset of the file piece to be transferred. |
type | Must be one of the following values:
|
flags | May be zero to indicate no special handling or any of the following or'd together:
|
Referenced by work_queue.Task::specify_file_piece().
int work_queue_task_specify_buffer | ( | struct work_queue_task * | t, |
const char * | data, | ||
int | length, | ||
const char * | remote_name, | ||
int | flags | ||
) |
Add an input buffer to a task.
t | A task object. |
data | The data to be passed as an input file. |
length | The length of the buffer, in bytes |
remote_name | The name of the remote file to create. |
flags | May be zero to indicate no special handling or any of the following or'd together:
|
Referenced by work_queue.Task::specify_buffer().
int work_queue_task_specify_directory | ( | struct work_queue_task * | t, |
const char * | local_name, | ||
const char * | remote_name, | ||
int | type, | ||
int | flags, | ||
int | recursive | ||
) |
Add a directory to a task.
t | A task object. |
local_name | The name of the directory on local disk or shared filesystem. Optional if the directory is empty. |
remote_name | The name of the directory at the remote execution site. |
type | Must be one of the following values:
|
flags | May be zero to indicate no special handling or any of the following or'd together:
|
recursive | indicates whether just the directory (0) or the directory and all of its contents (1) should be included. |
Referenced by work_queue.Task::specify_directory().
void work_queue_task_specify_memory | ( | struct work_queue_task * | t, |
int64_t | memory | ||
) |
Specify the amount of memory required by a task.
t | A task object. |
memory | The amount of memory required by the task, in megabytes. |
Referenced by work_queue.Task::specify_memory().
void work_queue_task_specify_disk | ( | struct work_queue_task * | t, |
int64_t | disk | ||
) |
Specify the amount of disk space required by a task.
t | A task object. |
disk | The amount of disk space required by the task, in megabytes. |
Referenced by work_queue.Task::specify_disk().
void work_queue_task_specify_cores | ( | struct work_queue_task * | t, |
int | cores | ||
) |
Specify the number of cores required by a task.
t | A task object. |
cores | The number of cores required by the task. |
Referenced by work_queue.Task::specify_cores().
void work_queue_task_specify_gpus | ( | struct work_queue_task * | t, |
int | gpus | ||
) |
Specify the number of gpus required by a task.
t | A task object. |
gpus | The number of gpus required by the task. |
void work_queue_task_specify_end_time | ( | struct work_queue_task * | t, |
int64_t | seconds | ||
) |
Specify the maximum end time allowed for the task (in seconds since the Epoch).
If seconds less than 1, then no end time is specified.
t | A task object. |
seconds | Number of seconds since the Epoch. |
Referenced by work_queue.Task::specify_end_time().
void work_queue_task_specify_tag | ( | struct work_queue_task * | t, |
const char * | tag | ||
) |
Attach a user defined string tag to the task.
This field is not interpreted by the work queue, but is provided for the user's convenience in identifying tasks when they complete.
t | A task object. |
tag | The tag to attach to task t. |
Referenced by work_queue.Task::specify_tag().
void work_queue_task_specify_priority | ( | struct work_queue_task * | t, |
double | priority | ||
) |
Specify the priority of this task relative to others in the queue.
Tasks with a higher priority value run first. If no priority is given, a task is placed at the end of the ready list, regardless of the priority.
t | A task object. |
priority | The priority of the task. |
Referenced by work_queue.Task::specify_end_time().
void work_queue_task_specify_algorithm | ( | struct work_queue_task * | t, |
int | algo | ||
) |
Select the scheduling algorithm for a single task.
To change the scheduling algorithm for all tasks, use work_queue_specify_algorithm instead.
t | A task object. |
algo | The algorithm to use in assigning this task to a worker:
|
Referenced by work_queue.Task::specify_algorithm().
void work_queue_task_delete | ( | struct work_queue_task * | t | ) |
Delete a task.
This may be called on tasks after they are returned from work_queue_wait.
t | The task to delete. |
Referenced by work_queue.Task::__init__().
struct work_queue* work_queue_create | ( | int | port | ) |
Create a new work queue.
Users may modify the behavior of work_queue_create by setting the following environmental variables before calling the function:
If the queue has a project name, then queue statistics and information will be reported to a catalog server. To specify the catalog server, the user may set the CATALOG_HOST and CATALOG_PORT environmental variables as described in catalog_query_create.
port | The port number to listen on. If zero is specified, then the port stored in the WORK_QUEUE_PORT environment variable is used if available. If it isn't, or if -1 is specified, the first unused port between WORK_QUEUE_LOW_PORT and WORK_QUEUE_HIGH_PORT (1024 and 32767 by default) is chosen. |
Referenced by work_queue.WorkQueue::__init__().
int work_queue_enable_monitoring | ( | struct work_queue * | q, |
char * | monitor_summary_file | ||
) |
Enables resource monitoring on the give work queue.
It generates the log file indicated by monitor_summary_file with all the summaries of the resources used by each task.
q | A work queue object. |
monitor_summary_file | The filename of the log (If NULL, it defaults to wq-pid-resource-usage). |
Referenced by work_queue.WorkQueue::enable_monitoring().
int work_queue_submit | ( | struct work_queue * | q, |
struct work_queue_task * | t | ||
) |
Submit a task to a queue.
Once a task is submitted to a queue, it is not longer under the user's control and should not be inspected until returned via work_queue_wait. Once returned, it is safe to re-submit the same take object via work_queue_submit.
q | A work queue object. |
t | A task object returned from work_queue_task_create. |
Referenced by work_queue.WorkQueue::submit().
void work_queue_blacklist_add | ( | struct work_queue * | q, |
const char * | hostname | ||
) |
Blacklist host from a queue.
q | A work queue object. |
hostname | A string for hostname. |
Referenced by work_queue.WorkQueue::blacklist().
void work_queue_blacklist_remove | ( | struct work_queue * | q, |
const char * | hostname | ||
) |
Unblacklist host from a queue.
q | A work queue object. |
hostname | A string for hostname. |
Referenced by work_queue.WorkQueue::blacklist_clear().
void work_queue_blacklist_clear | ( | struct work_queue * | q | ) |
Clear blacklist of a queue.
q | A work queue object. |
Referenced by work_queue.WorkQueue::blacklist_clear().
struct work_queue_task* work_queue_wait | ( | struct work_queue * | q, |
int | timeout | ||
) |
Wait for a task to complete.
This call will block until either a task has completed, the timeout has expired, or the queue is empty. If a task has completed, the corresponding task object will be returned by this function. The caller may examine the task and then dispose of it using work_queue_task_delete.
If the task ran to completion, then the result
field will be zero and the return_status
field will contain the Unix exit code of the task. If the task could not, then the result
field will be non-zero and the return_status
field will be undefined.
q | A work queue object. |
timeout | The number of seconds to wait for a completed task before returning. Use an integer time to set the timeout or the constant WORK_QUEUE_WAITFORTASK to block until a task has completed. |
Referenced by work_queue.WorkQueue::wait().
int work_queue_hungry | ( | struct work_queue * | q | ) |
Determine whether the queue is 'hungry' for more tasks.
While the Work Queue can handle a very large number of tasks, it runs most efficiently when the number of tasks is slightly larger than the number of active workers. This function gives the user of a flexible application a hint about whether it would be better to submit more tasks via work_queue_submit or wait for some to complete via work_queue_wait.
q | A work queue object. |
Referenced by work_queue.WorkQueue::hungry().
int work_queue_empty | ( | struct work_queue * | q | ) |
Determine whether the queue is empty.
When all of the desired tasks have been submitted to the queue, the user should continue to call work_queue_wait until this function returns true.
q | A work queue object. |
Referenced by work_queue.WorkQueue::empty().
int work_queue_port | ( | struct work_queue * | q | ) |
Get the listening port of the queue.
As noted in work_queue_create, there are many controls that affect what TCP port the queue will listen on. Rather than assuming a specific port, the user should simply call this function to determine what port was selected.
q | A work queue object. |
Referenced by work_queue.WorkQueue::port().
void work_queue_get_stats | ( | struct work_queue * | q, |
struct work_queue_stats * | s | ||
) |
Get queue statistics (only from master).
q | A work queue object. |
s | A pointer to a buffer that will be filed with statistics. |
Referenced by work_queue.WorkQueue::stats().
void work_queue_get_stats_hierarchy | ( | struct work_queue * | q, |
struct work_queue_stats * | s | ||
) |
Get statistics of the master queue together with foremen information.
q | A work queue object. |
s | A pointer to a buffer that will be filed with statistics. |
Referenced by work_queue.WorkQueue::stats_hierarchy().
int work_queue_task_state | ( | struct work_queue * | q, |
int | taskid | ||
) |
Get the current state of the task.
q | A work queue object. |
taskid | The taskid of the task. |
Referenced by work_queue.WorkQueue::task_state().
void work_queue_set_bandwidth_limit | ( | struct work_queue * | q, |
const char * | bandwidth | ||
) |
Limit the queue bandwidth when transferring files to and from workers.
q | A work queue object. |
bandwidth | The bandwidth limit in bytes per second. |
double work_queue_get_effective_bandwidth | ( | struct work_queue * | q | ) |
Get current queue bandwidth.
q | A work queue object. |
char* work_queue_get_worker_summary | ( | struct work_queue * | q | ) |
Summarize workers.
This function summarizes the workers currently connected to the master, indicating how many from each worker pool are attached.
q | A work queue object. |
int work_queue_activate_fast_abort | ( | struct work_queue * | q, |
double | multiplier | ||
) |
Turn on or off fast abort functionality for a given queue.
q | A work queue object. |
multiplier | The multiplier of the average task time at which point to abort; if negative (and by default) fast_abort is deactivated. |
Referenced by work_queue.WorkQueue::activate_fast_abort().
int work_queue_send_receive_ratio | ( | struct work_queue * | q, |
double | ratio | ||
) |
Change the preference to send or receive tasks.
q | A work queue object. |
ratio | The send/receive ratio when there is a choice between sending and receiving tasks. 1 Always prefer to send (e.g., for homogenous, stable resources). 0 Always prefer to receive (e.g., for resources with hight rate of eviction). Default is 0.75 (one average, receive one task per three sent). |
void work_queue_specify_algorithm | ( | struct work_queue * | q, |
int | algo | ||
) |
Change the worker selection algorithm.
This function controls which worker will be selected for a given task.
q | A work queue object. |
algo | The algorithm to use in assigning a task to a worker:
|
Referenced by work_queue.WorkQueue::specify_algorithm().
const char* work_queue_name | ( | struct work_queue * | q | ) |
Get the project name of the queue.
q | A work queue object. |
Referenced by work_queue.WorkQueue::name().
void work_queue_specify_name | ( | struct work_queue * | q, |
const char * | name | ||
) |
Change the project name for a given queue.
q | A work queue object. |
name | The new project name. |
Referenced by work_queue.WorkQueue::__init__(), and work_queue.WorkQueue::specify_name().
void work_queue_specify_priority | ( | struct work_queue * | q, |
int | priority | ||
) |
Change the priority for a given queue.
q | A work queue object. |
priority | The new priority of the queue. Higher priority masters will attract workers first. |
Referenced by work_queue.WorkQueue::specify_priority().
void work_queue_specify_catalog_server | ( | struct work_queue * | q, |
const char * | hostname, | ||
int | port | ||
) |
Specify the catalog server the master should report to.
q | A work queue object. |
hostname | The catalog server's hostname. |
port | The port the catalog server is listening on. |
Referenced by work_queue.WorkQueue::specify_catalog_server().
struct work_queue_task* work_queue_cancel_by_taskid | ( | struct work_queue * | q, |
int | id | ||
) |
Cancel a submitted task using its task id and remove it from queue.
q | A work queue object. |
id | The taskid returned from work_queue_submit. |
Referenced by work_queue.WorkQueue::cancel_by_taskid().
struct work_queue_task* work_queue_cancel_by_tasktag | ( | struct work_queue * | q, |
const char * | tag | ||
) |
Cancel a submitted task using its tag and remove it from queue.
q | A work queue object. |
tag | The tag name assigned to task using work_queue_task_specify_tag. |
Referenced by work_queue.WorkQueue::cancel_by_tasktag().
struct list* work_queue_cancel_all_tasks | ( | struct work_queue * | q | ) |
Cancel all submitted tasks and remove them from the queue.
q | A work queue object. |
int work_queue_shut_down_workers | ( | struct work_queue * | q, |
int | n | ||
) |
Shut down workers connected to the work_queue system.
Gives a best effort and then returns the number of workers given the shut down order.
q | A work queue object. |
n | The number to shut down. All workers if given "0". |
Referenced by work_queue.WorkQueue::shutdown_workers().
void work_queue_delete | ( | struct work_queue * | q | ) |
Delete a work queue.
This function should only be called after work_queue_empty returns true.
q | A work queue to delete. |
Referenced by work_queue.WorkQueue::__init__().
int work_queue_specify_log | ( | struct work_queue * | q, |
const char * | logfile | ||
) |
Add a log file that records the states of the connected workers and submitted tasks.
q | A work queue object. |
logfile | The filename. |
Referenced by work_queue.WorkQueue::specify_log().
void work_queue_specify_password | ( | struct work_queue * | q, |
const char * | password | ||
) |
Add a mandatory password that each worker must present.
q | A work queue object. |
password | The password to require. |
Referenced by work_queue.WorkQueue::specify_password().
int work_queue_specify_password_file | ( | struct work_queue * | q, |
const char * | file | ||
) |
Add a mandatory password file that each worker must present.
q | A work queue object. |
file | The name of the file containing the password. |
Referenced by work_queue.WorkQueue::specify_password_file().
void work_queue_specify_keepalive_interval | ( | struct work_queue * | q, |
int | interval | ||
) |
Change the keepalive interval for a given queue.
q | A work queue object. |
interval | The minimum number of seconds to wait before sending new keepalive checks to workers. |
Referenced by work_queue.WorkQueue::specify_keepalive_interval().
void work_queue_specify_keepalive_timeout | ( | struct work_queue * | q, |
int | timeout | ||
) |
Change the keepalive timeout for identifying dead workers for a given queue.
q | A work queue object. |
timeout | The minimum number of seconds to wait for a keepalive response from worker before marking it as dead. |
Referenced by work_queue.WorkQueue::specify_keepalive_timeout().
int work_queue_tune | ( | struct work_queue * | q, |
const char * | name, | ||
double | value | ||
) |
Tune advanced parameters for work queue.
q | A work queue object. |
name | The name of the parameter to tune
|
value | The value to set the parameter to. |
Referenced by work_queue.WorkQueue::tune().
void work_queue_specify_task_order | ( | struct work_queue * | q, |
int | order | ||
) |
Specified how the submitted tasks should be ordered.
It does not have any effect now.
q | A work queue object. |
order | The ordering to use for dispatching submitted tasks: |
Referenced by work_queue.WorkQueue::specify_task_order().
void work_queue_specify_master_mode | ( | struct work_queue * | q, |
int | mode | ||
) |
Specify the master mode for a given queue.
q | A work queue object. |
mode |
|
Referenced by work_queue.WorkQueue::__init__(), and work_queue.WorkQueue::specify_master_mode().
void work_queue_specify_estimate_capacity_on | ( | struct work_queue * | q, |
int | estimate_capacity_on | ||
) |
Change whether to estimate master capacity for a given queue.
q | A work queue object. |
estimate_capacity_on | if the value of this parameter is 1, then work queue should estimate the master capacity. If the value is 0, then work queue would not estimate its master capacity. |
Referenced by work_queue.WorkQueue::estimate_capacity().
int work_queue_task_specify_input_buf | ( | struct work_queue_task * | t, |
const char * | buf, | ||
int | length, | ||
const char * | rname | ||
) |
Add an input buffer to a task.
t | The task to which to add parameters |
buf | A pointer to the data buffer to send to the worker to be available to the commands. |
length | The number of bytes of data in the buffer |
rname | The name of the file in which to store the buffer data on the worker |
int work_queue_task_specify_input_file | ( | struct work_queue_task * | t, |
const char * | fname, | ||
const char * | rname | ||
) |
Add an input file to a task.
t | The task to which to add parameters |
fname | The name of the data file to send to the worker to be available to the commands. |
rname | The name of the file in which to store the buffer data on the worker. |
int work_queue_task_specify_input_file_do_not_cache | ( | struct work_queue_task * | t, |
const char * | fname, | ||
const char * | rname | ||
) |
Add an input file to a task, without caching.
t | The task to which to add parameters |
fname | The name of the data file to send to the worker to be available to the commands. |
rname | The name of the file in which to store the buffer data on the worker. |
int work_queue_task_specify_output_file | ( | struct work_queue_task * | t, |
const char * | rname, | ||
const char * | fname | ||
) |
Add an output file to a task.
t | The task to which to add parameters |
rname | The name of a file created by the program when it runs. |
fname | The name of the file local target for copying rname back. |
int work_queue_task_specify_output_file_do_not_cache | ( | struct work_queue_task * | t, |
const char * | rname, | ||
const char * | fname | ||
) |
Add an output file to a task without caching.
t | The task to which to add parameters |
rname | The name of a file created by the program when it runs. |
fname | The name of the file local target for copying rname back. |
double wq_option_fast_abort_multiplier |
Initial setting for fast abort multiplier upon creating queue.
Turned off if less than 0. Change prior to calling work_queue_create, after queue is created this variable is not considered and changes must be made through the API calls.
int wq_option_scheduler |
Initial setting for algorithm to assign tasks to workers upon creating queue .
Change prior to calling work_queue_create, after queue is created this variable is not considered and changes must be made through the API calls.