A master-worker library. More...
#include <sys/types.h>
#include "timestamp.h"
Go to the source code of this file.
Data Structures | |
struct | work_queue_task |
A task description. More... | |
struct | work_queue_stats |
Statistics describing a work queue. More... | |
Defines | |
#define | WORK_QUEUE_DEFAULT_PORT 9123 |
Default Work Queue port number. | |
#define | WORK_QUEUE_RANDOM_PORT 0 |
Indicates that any port number may be chosen. | |
#define | WORK_QUEUE_WAITFORTASK -1 |
Wait for a task to complete before returning. | |
#define | WORK_QUEUE_SCHEDULE_FCFS 1 |
Select worker on a first-come-first-serve basis. | |
#define | WORK_QUEUE_SCHEDULE_FILES 2 |
Select worker that has the most data required by the task. | |
#define | WORK_QUEUE_SCHEDULE_TIME 3 |
Select worker that has the fastest execution time on previous tasks. | |
#define | WORK_QUEUE_SCHEDULE_RAND 4 |
Select a random worker. | |
#define | WORK_QUEUE_TASK_ORDER_FIFO 0 |
Retrieve tasks based on first-in-first-out order. | |
#define | WORK_QUEUE_TASK_ORDER_LIFO 1 |
Retrieve tasks based on last-in-first-out order. | |
#define | WORK_QUEUE_INPUT 0 |
Specify an input object. | |
#define | WORK_QUEUE_OUTPUT 1 |
Specify an output object. | |
#define | WORK_QUEUE_NOCACHE 0 |
Do not cache file at execution site. | |
#define | WORK_QUEUE_CACHE 1 |
Cache file at execution site for later use. | |
#define | WORK_QUEUE_WATCH 16 |
Watch the output file and send back changes as the task runs. | |
#define | WORK_QUEUE_RESET_ALL 0 |
When resetting, clear out all tasks and files. | |
#define | WORK_QUEUE_RESET_KEEP_TASKS 1 |
When resetting, keep the current list of tasks. | |
#define | WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300 |
Default value for Work Queue keepalive interval in seconds. | |
#define | WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30 |
Default value for Work Queue keepalive timeout in seconds. | |
#define | WORK_QUEUE_RESULT_SUCCESS 0 |
The task ran successfully >. | |
#define | WORK_QUEUE_RESULT_INPUT_MISSING 1 |
The task cannot be run due to a missing input file >. | |
#define | WORK_QUEUE_RESULT_OUTPUT_MISSING 2 |
The task ran but failed to generate a specified output file >. | |
#define | WORK_QUEUE_RESULT_STDOUT_MISSING 4 |
The task ran but its stdout has been truncated >. | |
#define | WORK_QUEUE_RESULT_SIGNAL 8 |
The task was terminated with a signal >. | |
#define | WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 |
The task used more resources than requested >. | |
#define | WORK_QUEUE_RESULT_TASK_TIMEOUT 32 |
The task ran after specified end time. | |
Functions | |
Functions - Tasks | |
struct work_queue_task * | work_queue_task_create (const char *full_command) |
Create a new task object. | |
struct work_queue_task * | work_queue_task_clone (const struct work_queue_task *task) |
Create a copy of a task Create a functionally identical copy of a work_queue_task that can be re-submitted via work_queue_submit. | |
void | work_queue_task_specify_command (struct work_queue_task *t, const char *cmd) |
Indicate the command to be executed. | |
int | work_queue_task_specify_file (struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags) |
Add a file to a task. | |
int | work_queue_task_specify_file_piece (struct work_queue_task *t, const char *local_name, const char *remote_name, off_t start_byte, off_t end_byte, int type, int flags) |
Add a file piece to a task. | |
int | work_queue_task_specify_buffer (struct work_queue_task *t, const char *data, int length, const char *remote_name, int flags) |
Add an input buffer to a task. | |
int | work_queue_task_specify_directory (struct work_queue_task *t, const char *local_name, const char *remote_name, int type, int flags, int recursive) |
Add a directory to a task. | |
void | work_queue_task_specify_memory (struct work_queue_task *t, int64_t memory) |
Specify the amount of memory required by a task. | |
void | work_queue_task_specify_disk (struct work_queue_task *t, int64_t disk) |
Specify the amount of disk space required by a task. | |
void | work_queue_task_specify_cores (struct work_queue_task *t, int cores) |
Specify the number of cores required by a task. | |
void | work_queue_task_specify_gpus (struct work_queue_task *t, int gpus) |
Specify the number of gpus required by a task. | |
void | work_queue_task_specify_end_time (struct work_queue_task *t, int64_t seconds) |
Specify the maximum end time allowed for the task (in seconds since the Epoch). | |
void | work_queue_task_specify_tag (struct work_queue_task *t, const char *tag) |
Attach a user defined string tag to the task. | |
void | work_queue_task_specify_algorithm (struct work_queue_task *t, int algo) |
Select the scheduling algorithm for a single task. | |
void | work_queue_task_delete (struct work_queue_task *t) |
Delete a task. | |
Functions - Queues | |
struct work_queue * | work_queue_create (int port) |
Create a new work queue. | |
int | work_queue_enable_monitoring (struct work_queue *q, char *monitor_summary_file) |
Enables resource monitoring on the give work queue. | |
int | work_queue_submit (struct work_queue *q, struct work_queue_task *t) |
Submit a task to a queue. | |
void | work_queue_blacklist_add (struct work_queue *q, const char *hostname) |
Blacklist host from a queue. | |
void | work_queue_blacklist_remove (struct work_queue *q, const char *hostname) |
Unblacklist host from a queue. | |
void | work_queue_blacklist_clear (struct work_queue *q) |
Clear blacklist of a queue. | |
struct work_queue_task * | work_queue_wait (struct work_queue *q, int timeout) |
Wait for a task to complete. | |
int | work_queue_hungry (struct work_queue *q) |
Determine whether the queue is 'hungry' for more tasks. | |
int | work_queue_empty (struct work_queue *q) |
Determine whether the queue is empty. | |
int | work_queue_port (struct work_queue *q) |
Get the listening port of the queue. | |
void | work_queue_get_stats (struct work_queue *q, struct work_queue_stats *s) |
Get queue statistics (only from master). | |
void | work_queue_get_stats_hierarchy (struct work_queue *q, struct work_queue_stats *s) |
Get statistics of the master queue together with foremen information. | |
void | work_queue_set_bandwidth_limit (struct work_queue *q, const char *bandwidth) |
Limit the queue bandwidth when transferring files to and from workers. | |
double | work_queue_get_effective_bandwidth (struct work_queue *q) |
Get current queue bandwidth. | |
char * | work_queue_get_worker_summary (struct work_queue *q) |
Summarize workers. | |
int | work_queue_activate_fast_abort (struct work_queue *q, double multiplier) |
Turn on or off fast abort functionality for a given queue. | |
int | work_queue_send_receive_ratio (struct work_queue *q, double ratio) |
Change the preference to send or receive tasks. | |
void | work_queue_specify_algorithm (struct work_queue *q, int algo) |
Change the worker selection algorithm. | |
void | work_queue_specify_task_order (struct work_queue *q, int order) |
Specify how the submitted tasks should be ordered. | |
const char * | work_queue_name (struct work_queue *q) |
Get the project name of the queue. | |
void | work_queue_specify_name (struct work_queue *q, const char *name) |
Change the project name for a given queue. | |
void | work_queue_specify_priority (struct work_queue *q, int priority) |
Change the priority for a given queue. | |
void | work_queue_specify_catalog_server (struct work_queue *q, const char *hostname, int port) |
Specify the catalog server the master should report to. | |
struct work_queue_task * | work_queue_cancel_by_taskid (struct work_queue *q, int id) |
Cancel a submitted task using its task id and remove it from queue. | |
struct work_queue_task * | work_queue_cancel_by_tasktag (struct work_queue *q, const char *tag) |
Cancel a submitted task using its tag and remove it from queue. | |
struct list * | work_queue_cancel_all_tasks (struct work_queue *q) |
Cancel all submitted tasks and remove them from the queue. | |
int | work_queue_shut_down_workers (struct work_queue *q, int n) |
Shut down workers connected to the work_queue system. | |
void | work_queue_delete (struct work_queue *q) |
Delete a work queue. | |
int | work_queue_specify_log (struct work_queue *q, const char *logfile) |
Add a log file that records the states of the connected workers and submitted tasks. | |
void | work_queue_specify_password (struct work_queue *q, const char *password) |
Add a mandatory password that each worker must present. | |
int | work_queue_specify_password_file (struct work_queue *q, const char *file) |
Add a mandatory password file that each worker must present. | |
void | work_queue_specify_keepalive_interval (struct work_queue *q, int interval) |
Change the keepalive interval for a given queue. | |
void | work_queue_specify_keepalive_timeout (struct work_queue *q, int timeout) |
Change the keepalive timeout for identifying dead workers for a given queue. | |
int | work_queue_tune (struct work_queue *q, const char *name, double value) |
Tune advanced parameters for work queue. | |
Variables | |
double | wq_option_fast_abort_multiplier |
Initial setting for fast abort multiplier upon creating queue. | |
int | wq_option_scheduler |
Initial setting for algorithm to assign tasks to workers upon creating queue . | |
Functions - Deprecated | |
| |
#define | WORK_QUEUE_MASTER_MODE_STANDALONE 0 |
Work Queue master does not report to the catalog server. | |
#define | WORK_QUEUE_MASTER_MODE_CATALOG 1 |
Work Queue master reports to catalog server. | |
void | work_queue_specify_master_mode (struct work_queue *q, int mode) |
Specify the master mode for a given queue. | |
void | work_queue_specify_estimate_capacity_on (struct work_queue *q, int estimate_capacity_on) |
Change whether to estimate master capacity for a given queue. | |
int | work_queue_task_specify_input_buf (struct work_queue_task *t, const char *buf, int length, const char *rname) |
Add an input buffer to a task. | |
int | work_queue_task_specify_input_file (struct work_queue_task *t, const char *fname, const char *rname) |
Add an input file to a task. | |
int | work_queue_task_specify_input_file_do_not_cache (struct work_queue_task *t, const char *fname, const char *rname) |
Add an input file to a task, without caching. | |
int | work_queue_task_specify_output_file (struct work_queue_task *t, const char *rname, const char *fname) |
Add an output file to a task. | |
int | work_queue_task_specify_output_file_do_not_cache (struct work_queue_task *t, const char *rname, const char *fname) |
Add an output file to a task without caching. |
A master-worker library.
The work queue provides an implementation of the master-worker computing model using TCP sockets, Unix applications, and files as intermediate buffers. A master process uses work_queue_create to create a queue, then work_queue_submit to submit tasks. Once tasks are running, call work_queue_wait to wait for completion. A generic worker program, named work_queue_worker
, can be run on any machine, and simply needs to be told the host and port of the master.
#define WORK_QUEUE_DEFAULT_PORT 9123 |
Default Work Queue port number.
#define WORK_QUEUE_RANDOM_PORT 0 |
Indicates that any port number may be chosen.
#define WORK_QUEUE_WAITFORTASK -1 |
Wait for a task to complete before returning.
#define WORK_QUEUE_SCHEDULE_FCFS 1 |
Select worker on a first-come-first-serve basis.
#define WORK_QUEUE_SCHEDULE_FILES 2 |
Select worker that has the most data required by the task.
#define WORK_QUEUE_SCHEDULE_TIME 3 |
Select worker that has the fastest execution time on previous tasks.
#define WORK_QUEUE_SCHEDULE_RAND 4 |
Select a random worker.
(default)
#define WORK_QUEUE_TASK_ORDER_FIFO 0 |
Retrieve tasks based on first-in-first-out order.
#define WORK_QUEUE_TASK_ORDER_LIFO 1 |
Retrieve tasks based on last-in-first-out order.
#define WORK_QUEUE_INPUT 0 |
Specify an input object.
#define WORK_QUEUE_OUTPUT 1 |
Specify an output object.
#define WORK_QUEUE_NOCACHE 0 |
Do not cache file at execution site.
#define WORK_QUEUE_CACHE 1 |
Cache file at execution site for later use.
#define WORK_QUEUE_WATCH 16 |
Watch the output file and send back changes as the task runs.
#define WORK_QUEUE_RESET_ALL 0 |
When resetting, clear out all tasks and files.
#define WORK_QUEUE_RESET_KEEP_TASKS 1 |
When resetting, keep the current list of tasks.
#define WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL 300 |
Default value for Work Queue keepalive interval in seconds.
#define WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT 30 |
Default value for Work Queue keepalive timeout in seconds.
#define WORK_QUEUE_RESULT_SUCCESS 0 |
The task ran successfully >.
#define WORK_QUEUE_RESULT_INPUT_MISSING 1 |
The task cannot be run due to a missing input file >.
#define WORK_QUEUE_RESULT_OUTPUT_MISSING 2 |
The task ran but failed to generate a specified output file >.
#define WORK_QUEUE_RESULT_STDOUT_MISSING 4 |
The task ran but its stdout has been truncated >.
#define WORK_QUEUE_RESULT_SIGNAL 8 |
The task was terminated with a signal >.
#define WORK_QUEUE_RESULT_RESOURCE_EXHAUSTION 16 |
The task used more resources than requested >.
#define WORK_QUEUE_RESULT_TASK_TIMEOUT 32 |
The task ran after specified end time.
>
#define WORK_QUEUE_MASTER_MODE_STANDALONE 0 |
Work Queue master does not report to the catalog server.
#define WORK_QUEUE_MASTER_MODE_CATALOG 1 |
Work Queue master reports to catalog server.
struct work_queue_task* work_queue_task_create | ( | const char * | full_command | ) | [read] |
Create a new task object.
Once created and elaborated with functions such as work_queue_task_specify_file and work_queue_task_specify_buffer, the task should be passed to work_queue_submit.
full_command | The shell command line to be executed by the task. If null, the command will be given later by work_queue_task_specify_command |
struct work_queue_task* work_queue_task_clone | ( | const struct work_queue_task * | task | ) | [read] |
Create a copy of a task Create a functionally identical copy of a work_queue_task that can be re-submitted via work_queue_submit.
void work_queue_task_specify_command | ( | struct work_queue_task * | t, | |
const char * | cmd | |||
) |
Indicate the command to be executed.
t | A task object. | |
cmd | The command to be executed. This string will be duplicated by this call, so the argument may be freed or re-used afterward. |
int work_queue_task_specify_file | ( | struct work_queue_task * | t, | |
const char * | local_name, | |||
const char * | remote_name, | |||
int | type, | |||
int | flags | |||
) |
Add a file to a task.
t | A task object. | |
local_name | The name of the file on local disk or shared filesystem. | |
remote_name | The name of the file at the remote execution site. | |
type | Must be one of the following values:
| |
flags | May be zero to indicate no special handling or any of the following or'd together:
|
int work_queue_task_specify_file_piece | ( | struct work_queue_task * | t, | |
const char * | local_name, | |||
const char * | remote_name, | |||
off_t | start_byte, | |||
off_t | end_byte, | |||
int | type, | |||
int | flags | |||
) |
Add a file piece to a task.
t | A task object. | |
local_name | The name of the file on local disk or shared filesystem. | |
remote_name | The name of the file at the remote execution site. | |
start_byte | The starting byte offset of the file piece to be transferred. | |
end_byte | The ending byte offset of the file piece to be transferred. | |
type | Must be one of the following values:
| |
flags | May be zero to indicate no special handling or any of the following or'd together:
|
int work_queue_task_specify_buffer | ( | struct work_queue_task * | t, | |
const char * | data, | |||
int | length, | |||
const char * | remote_name, | |||
int | flags | |||
) |
Add an input buffer to a task.
t | A task object. | |
data | The data to be passed as an input file. | |
length | The length of the buffer, in bytes | |
remote_name | The name of the remote file to create. | |
flags | May be zero to indicate no special handling or any of the following or'd together:
|
int work_queue_task_specify_directory | ( | struct work_queue_task * | t, | |
const char * | local_name, | |||
const char * | remote_name, | |||
int | type, | |||
int | flags, | |||
int | recursive | |||
) |
Add a directory to a task.
t | A task object. | |
local_name | The name of the directory on local disk or shared filesystem. Optional if the directory is empty. | |
remote_name | The name of the directory at the remote execution site. | |
type | Must be one of the following values:
| |
flags | May be zero to indicate no special handling or any of the following or'd together:
| |
recursive | indicates whether just the directory (0) or the directory and all of its contents (1) should be included. |
void work_queue_task_specify_memory | ( | struct work_queue_task * | t, | |
int64_t | memory | |||
) |
Specify the amount of memory required by a task.
t | A task object. | |
memory | The amount of memory required by the task, in megabytes. |
void work_queue_task_specify_disk | ( | struct work_queue_task * | t, | |
int64_t | disk | |||
) |
Specify the amount of disk space required by a task.
t | A task object. | |
disk | The amount of disk space required by the task, in megabytes. |
void work_queue_task_specify_cores | ( | struct work_queue_task * | t, | |
int | cores | |||
) |
Specify the number of cores required by a task.
t | A task object. | |
cores | The number of cores required by the task. |
void work_queue_task_specify_gpus | ( | struct work_queue_task * | t, | |
int | gpus | |||
) |
Specify the number of gpus required by a task.
t | A task object. | |
gpus | The number of gpus required by the task. |
void work_queue_task_specify_end_time | ( | struct work_queue_task * | t, | |
int64_t | seconds | |||
) |
Specify the maximum end time allowed for the task (in seconds since the Epoch).
If seconds less than 1, then no end time is specified.
t | A task object. | |
seconds | Number of seconds since the Epoch. |
void work_queue_task_specify_tag | ( | struct work_queue_task * | t, | |
const char * | tag | |||
) |
Attach a user defined string tag to the task.
This field is not interpreted by the work queue, but is provided for the user's convenience in identifying tasks when they complete.
t | A task object. | |
tag | The tag to attach to task t. |
void work_queue_task_specify_algorithm | ( | struct work_queue_task * | t, | |
int | algo | |||
) |
Select the scheduling algorithm for a single task.
To change the scheduling algorithm for all tasks, use work_queue_specify_algorithm instead.
t | A task object. | |
algo | The algorithm to use in assigning this task to a worker:
|
void work_queue_task_delete | ( | struct work_queue_task * | t | ) |
Delete a task.
This may be called on tasks after they are returned from work_queue_wait.
t | The task to delete. |
struct work_queue* work_queue_create | ( | int | port | ) | [read] |
Create a new work queue.
Users may modify the behavior of work_queue_create by setting the following environmental variables before calling the function:
If the queue has a project name, then queue statistics and information will be reported to a catalog server. To specify the catalog server, the user may set the CATALOG_HOST and CATALOG_PORT environmental variables as described in catalog_query_create.
port | The port number to listen on. If zero is specified, then the port stored in the WORK_QUEUE_PORT environment variable is used if available. If it isn't, or if -1 is specified, the first unused port between WORK_QUEUE_LOW_PORT and WORK_QUEUE_HIGH_PORT (1024 and 32767 by default) is chosen. |
int work_queue_enable_monitoring | ( | struct work_queue * | q, | |
char * | monitor_summary_file | |||
) |
Enables resource monitoring on the give work queue.
It generates the log file indicated by monitor_summary_file with all the summaries of the resources used by each task.
q | A work queue object. | |
monitor_summary_file | The filename of the log (If NULL, it defaults to wq-pid-resource-usage). |
int work_queue_submit | ( | struct work_queue * | q, | |
struct work_queue_task * | t | |||
) |
Submit a task to a queue.
Once a task is submitted to a queue, it is not longer under the user's control and should not be inspected until returned via work_queue_wait. Once returned, it is safe to re-submit the same take object via work_queue_submit.
q | A work queue object. | |
t | A task object returned from work_queue_task_create. |
void work_queue_blacklist_add | ( | struct work_queue * | q, | |
const char * | hostname | |||
) |
Blacklist host from a queue.
q | A work queue object. | |
hostname | A string for hostname. |
void work_queue_blacklist_remove | ( | struct work_queue * | q, | |
const char * | hostname | |||
) |
Unblacklist host from a queue.
q | A work queue object. | |
hostname | A string for hostname. |
void work_queue_blacklist_clear | ( | struct work_queue * | q | ) |
Clear blacklist of a queue.
q | A work queue object. |
struct work_queue_task* work_queue_wait | ( | struct work_queue * | q, | |
int | timeout | |||
) | [read] |
Wait for a task to complete.
This call will block until either a task has completed, the timeout has expired, or the queue is empty. If a task has completed, the corresponding task object will be returned by this function. The caller may examine the task and then dispose of it using work_queue_task_delete.
If the task ran to completion, then the result
field will be zero and the return_status
field will contain the Unix exit code of the task. If the task could not, then the result
field will be non-zero and the return_status
field will be undefined.
q | A work queue object. | |
timeout | The number of seconds to wait for a completed task before returning. Use an integer time to set the timeout or the constant WORK_QUEUE_WAITFORTASK to block until a task has completed. |
int work_queue_hungry | ( | struct work_queue * | q | ) |
Determine whether the queue is 'hungry' for more tasks.
While the Work Queue can handle a very large number of tasks, it runs most efficiently when the number of tasks is slightly larger than the number of active workers. This function gives the user of a flexible application a hint about whether it would be better to submit more tasks via work_queue_submit or wait for some to complete via work_queue_wait.
q | A work queue object. |
int work_queue_empty | ( | struct work_queue * | q | ) |
Determine whether the queue is empty.
When all of the desired tasks have been submitted to the queue, the user should continue to call work_queue_wait until this function returns true.
q | A work queue object. |
int work_queue_port | ( | struct work_queue * | q | ) |
Get the listening port of the queue.
As noted in work_queue_create, there are many controls that affect what TCP port the queue will listen on. Rather than assuming a specific port, the user should simply call this function to determine what port was selected.
q | A work queue object. |
void work_queue_get_stats | ( | struct work_queue * | q, | |
struct work_queue_stats * | s | |||
) |
Get queue statistics (only from master).
q | A work queue object. | |
s | A pointer to a buffer that will be filed with statistics. |
void work_queue_get_stats_hierarchy | ( | struct work_queue * | q, | |
struct work_queue_stats * | s | |||
) |
Get statistics of the master queue together with foremen information.
q | A work queue object. | |
s | A pointer to a buffer that will be filed with statistics. |
void work_queue_set_bandwidth_limit | ( | struct work_queue * | q, | |
const char * | bandwidth | |||
) |
Limit the queue bandwidth when transferring files to and from workers.
q | A work queue object. | |
bandwidth | The bandwidth limit in bytes per second. |
double work_queue_get_effective_bandwidth | ( | struct work_queue * | q | ) |
Get current queue bandwidth.
q | A work queue object. |
char* work_queue_get_worker_summary | ( | struct work_queue * | q | ) |
Summarize workers.
This function summarizes the workers currently connected to the master, indicating how many from each worker pool are attached.
q | A work queue object. |
int work_queue_activate_fast_abort | ( | struct work_queue * | q, | |
double | multiplier | |||
) |
Turn on or off fast abort functionality for a given queue.
q | A work queue object. | |
multiplier | The multiplier of the average task time at which point to abort; if negative (and by default) fast_abort is deactivated. |
int work_queue_send_receive_ratio | ( | struct work_queue * | q, | |
double | ratio | |||
) |
Change the preference to send or receive tasks.
q | A work queue object. | |
ratio | The send/receive ratio when there is a choice between sending and receiving tasks. 1 Always prefer to send (e.g., for homogenous, stable resources). 0 Always prefer to receive (e.g., for resources with hight rate of eviction). Default is 0.75 (one average, receive one task per three sent). |
void work_queue_specify_algorithm | ( | struct work_queue * | q, | |
int | algo | |||
) |
Change the worker selection algorithm.
Note that this function controls which worker will be selected for a given task while work_queue_specify_task_order controls which task will be executed next.
q | A work queue object. | |
algo | The algorithm to use in assigning a task to a worker:
|
void work_queue_specify_task_order | ( | struct work_queue * | q, | |
int | order | |||
) |
Specify how the submitted tasks should be ordered.
Note that this function controls which task to execute next, while work_queue_specify_algorithm controls which worker it should be assigned to.
q | A work queue object. | |
order | The ordering to use for dispatching submitted tasks: |
const char* work_queue_name | ( | struct work_queue * | q | ) |
Get the project name of the queue.
q | A work queue object. |
void work_queue_specify_name | ( | struct work_queue * | q, | |
const char * | name | |||
) |
Change the project name for a given queue.
q | A work queue object. | |
name | The new project name. |
void work_queue_specify_priority | ( | struct work_queue * | q, | |
int | priority | |||
) |
Change the priority for a given queue.
q | A work queue object. | |
priority | The new priority of the queue. Higher priority masters will attract workers first. |
void work_queue_specify_catalog_server | ( | struct work_queue * | q, | |
const char * | hostname, | |||
int | port | |||
) |
Specify the catalog server the master should report to.
q | A work queue object. | |
hostname | The catalog server's hostname. | |
port | The port the catalog server is listening on. |
struct work_queue_task* work_queue_cancel_by_taskid | ( | struct work_queue * | q, | |
int | id | |||
) | [read] |
Cancel a submitted task using its task id and remove it from queue.
q | A work queue object. | |
id | The taskid returned from work_queue_submit. |
struct work_queue_task* work_queue_cancel_by_tasktag | ( | struct work_queue * | q, | |
const char * | tag | |||
) | [read] |
Cancel a submitted task using its tag and remove it from queue.
q | A work queue object. | |
tag | The tag name assigned to task using work_queue_task_specify_tag. |
struct list* work_queue_cancel_all_tasks | ( | struct work_queue * | q | ) | [read] |
Cancel all submitted tasks and remove them from the queue.
q | A work queue object. |
int work_queue_shut_down_workers | ( | struct work_queue * | q, | |
int | n | |||
) |
Shut down workers connected to the work_queue system.
Gives a best effort and then returns the number of workers given the shut down order.
q | A work queue object. | |
n | The number to shut down. All workers if given "0". |
void work_queue_delete | ( | struct work_queue * | q | ) |
Delete a work queue.
This function should only be called after work_queue_empty returns true.
q | A work queue to delete. |
int work_queue_specify_log | ( | struct work_queue * | q, | |
const char * | logfile | |||
) |
Add a log file that records the states of the connected workers and submitted tasks.
q | A work queue object. | |
logfile | The filename. |
void work_queue_specify_password | ( | struct work_queue * | q, | |
const char * | password | |||
) |
Add a mandatory password that each worker must present.
q | A work queue object. | |
password | The password to require. |
int work_queue_specify_password_file | ( | struct work_queue * | q, | |
const char * | file | |||
) |
Add a mandatory password file that each worker must present.
q | A work queue object. | |
file | The name of the file containing the password. |
void work_queue_specify_keepalive_interval | ( | struct work_queue * | q, | |
int | interval | |||
) |
Change the keepalive interval for a given queue.
q | A work queue object. | |
interval | The minimum number of seconds to wait before sending new keepalive checks to workers. |
void work_queue_specify_keepalive_timeout | ( | struct work_queue * | q, | |
int | timeout | |||
) |
Change the keepalive timeout for identifying dead workers for a given queue.
q | A work queue object. | |
timeout | The minimum number of seconds to wait for a keepalive response from worker before marking it as dead. |
int work_queue_tune | ( | struct work_queue * | q, | |
const char * | name, | |||
double | value | |||
) |
Tune advanced parameters for work queue.
q | A work queue object. | |
name | The name of the parameter to tune
| |
value | The value to set the parameter to. |
void work_queue_specify_master_mode | ( | struct work_queue * | q, | |
int | mode | |||
) |
Specify the master mode for a given queue.
q | A work queue object. | |
mode |
|
void work_queue_specify_estimate_capacity_on | ( | struct work_queue * | q, | |
int | estimate_capacity_on | |||
) |
Change whether to estimate master capacity for a given queue.
q | A work queue object. | |
estimate_capacity_on | if the value of this parameter is 1, then work queue should estimate the master capacity. If the value is 0, then work queue would not estimate its master capacity. |
int work_queue_task_specify_input_buf | ( | struct work_queue_task * | t, | |
const char * | buf, | |||
int | length, | |||
const char * | rname | |||
) |
Add an input buffer to a task.
t | The task to which to add parameters | |
buf | A pointer to the data buffer to send to the worker to be available to the commands. | |
length | The number of bytes of data in the buffer | |
rname | The name of the file in which to store the buffer data on the worker |
int work_queue_task_specify_input_file | ( | struct work_queue_task * | t, | |
const char * | fname, | |||
const char * | rname | |||
) |
Add an input file to a task.
t | The task to which to add parameters | |
fname | The name of the data file to send to the worker to be available to the commands. | |
rname | The name of the file in which to store the buffer data on the worker. |
int work_queue_task_specify_input_file_do_not_cache | ( | struct work_queue_task * | t, | |
const char * | fname, | |||
const char * | rname | |||
) |
Add an input file to a task, without caching.
t | The task to which to add parameters | |
fname | The name of the data file to send to the worker to be available to the commands. | |
rname | The name of the file in which to store the buffer data on the worker. |
int work_queue_task_specify_output_file | ( | struct work_queue_task * | t, | |
const char * | rname, | |||
const char * | fname | |||
) |
Add an output file to a task.
t | The task to which to add parameters | |
rname | The name of a file created by the program when it runs. | |
fname | The name of the file local target for copying rname back. |
int work_queue_task_specify_output_file_do_not_cache | ( | struct work_queue_task * | t, | |
const char * | rname, | |||
const char * | fname | |||
) |
Add an output file to a task without caching.
t | The task to which to add parameters | |
rname | The name of a file created by the program when it runs. | |
fname | The name of the file local target for copying rname back. |
Initial setting for fast abort multiplier upon creating queue.
Turned off if less than 0. Change prior to calling work_queue_create, after queue is created this variable is not considered and changes must be made through the API calls.
Initial setting for algorithm to assign tasks to workers upon creating queue .
Change prior to calling work_queue_create, after queue is created this variable is not considered and changes must be made through the API calls.