org.eclipse.smila.jobmanager.persistence

Interface RunStorage

All Known Implementing Classes:

RunStorageZk

public interface RunStorage

Persistence component interface for run data.

Method Summary

Methods

Modifier and Type	Method and Description
void	addBulkForBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position, java.lang.String bulkId)
void	addJobTrigger(java.lang.String bucketId, java.lang.String jobName)
void	addTaskForBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position, java.lang.String taskId)
void	addTransientBulk(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String bulk) Stores a transient bulk for the given workflow run.
java.util.List<java.lang.String>	cancelJobRun(java.lang.String jobName, java.lang.String jobRunId) Cancels the given job run data -> active workflow runs and current tasks will be canceled.
boolean	checkAndCleanupActiveWorkflowRuns(java.lang.String jobName, java.lang.String jobRunId) Checks if there are active workflow runs.
void	cleanOpenedBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position)
void	clear() Remove all jobmanager run data.
void	deleteJobRun(java.lang.String jobName, java.lang.String jobRunId) Removes the job run data of the given job run.
void	deleteWorkflowRun(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId) Deletes the given workflow run.
void	deleteWorkflowRunData(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId) Deletes the data of the given workflow run.
void	failedTask(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String stepId, java.lang.String taskId, boolean failedAfterRetry, java.util.Map<java.lang.String,java.lang.String> properties) Called when a task is failed.
void	failedWorkflowRun(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId) Called when a workflow run failed.
boolean	finishJobRun(java.lang.String jobName, java.lang.String jobRunId) Called when a job run is finished.
void	finishTask(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String stepId, java.lang.String taskId, java.util.Map<java.lang.String,java.lang.Number> workerCounter, java.util.Map<java.lang.String,java.lang.String> properties) Called when a task is successfully finished.
java.util.Collection<java.lang.String>	getBulksForBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position)
java.util.Collection<java.lang.String>	getCurrentJobs()
java.lang.String	getCurrentTimestamp()
byte[]	getCustomData(java.lang.String jobName, java.lang.String worker, java.lang.String... path)
AnyMap	getJobRunData(java.lang.String jobName, boolean returnDetails)
java.lang.String	getJobRunId(java.lang.String jobName)
JobRunInfo	getJobRunInfo(java.lang.String jobName)
JobState	getJobState(java.lang.String jobName)
JobState	getJobState(java.lang.String jobName, java.lang.String jobRunId)
int	getTaskRetries(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String stepId, java.lang.String taskId) get the number of retries for the task.
java.util.Collection<java.lang.String>	getTransientBulks(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId)
java.util.Collection<java.lang.String>	getTriggeredJobs(java.lang.String bucketId)
AnyMap	getWorkflowRunData(java.lang.String jobName, java.lang.String workflowRunId) get information about a workflow run.
boolean	hasTask(java.lang.String jobName, java.lang.String workflowRunId, java.lang.String taskId) Check if the given task with the given task id exists for the job and workflowrun.
boolean	hasTasks(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId)
boolean	hasWorkflowRun(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId)
void	obsoleteTask(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String stepId, java.lang.String taskId, java.util.Map<java.lang.String,java.lang.String> properties) Called when a task finished as obsolete.
boolean	prepareOpeningOfBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position)
boolean	prepareToFinishWorkflowRun(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId) Called before a workflow run is finished and deleted.
void	registerJobRunListener(JobRunListener jobRunListener, java.lang.String jobName)
void	removeJobTrigger(java.lang.String bucketId, java.lang.String jobName)
void	removeTaskForBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position, java.lang.String taskId)
void	retriedTask(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String stepId, java.lang.String taskId, boolean retryByWorker, java.util.Map<java.lang.String,java.lang.String> properties) Called when a task is retried.
void	setCustomData(java.lang.String jobName, java.lang.String worker, byte[] data, java.lang.String... path) Store custom data for given job, worker and path.
void	setJobState(java.lang.String jobName, java.lang.String jobRunId, JobState jobState) Sets the state for the given job run.
boolean	setJobState(java.lang.String jobName, java.lang.String jobRunId, JobState expectedState, JobState newState) Sets the state for the given job run if the given expected state matches the current state.
void	setupBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position)
java.lang.String	startCompletionWorkflowRun(java.lang.String jobName, java.lang.String jobRunId) Called when a new completing workflow run is started for the given job run.
void	startJobRun(java.lang.String jobName, java.lang.String jobRunId, JobRunMode jobRunMode, JobRunDefinitions jobRunDefs) Starts a job run.
void	startTask(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String stepId, java.lang.String taskId) Called when a new task is started for the given step in the given workflow of the given job run.
void	startTask(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.lang.String stepId, java.lang.String taskId, int numberOfRetries) Called when a new task is started for the given step in the given workflow of the given job run.
void	startTasks(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId, java.util.Collection<Task> tasks) Start multiple new tasks for a given job/workflow run at once.
java.lang.String	startWorkflowRun(java.lang.String jobName, java.lang.String jobRunId) Called when a new workflow run is started for the given job run.
void	successfulWorkflowRun(java.lang.String jobName, java.lang.String jobRunId, java.lang.String workflowRunId) Called when a workflow run successfully finished.
boolean	tasksLockBarrier(java.lang.String jobName, java.lang.String workflowRunId, int position)

Method Detail

clear

void clear()
           throws RunStorageException

Remove all jobmanager run data. Can be called concurrently.

Throws:

RunStorageException - error

startJobRun

void startJobRun(java.lang.String jobName,
               java.lang.String jobRunId,
               JobRunMode jobRunMode,
               JobRunDefinitions jobRunDefs)
                 throws RunStorageException

Starts a job run. If another instance already started the job the method will throw an exception.

Parameters:

jobName - The name of the job to start.

jobRunId - The job run id for the started job run.

jobRunMode - The job run mode for the started job run.

jobRunDefs - The job run definitions that are stored for this job run.

Throws:

RunStorageException - If there's an error while accessing the run storage.

finishJobRun

boolean finishJobRun(java.lang.String jobName,
                   java.lang.String jobRunId)
                     throws RunStorageException

Called when a job run is finished. This does _not_ remove the job run data. Since it is possible that more instances of a job manager try to finish a job run concurrently the one that succeeded will get 'true' as return value, the others false but no exception will be thrown in that case, because this is not an exceptional behavior.

Parameters:

jobName - The job name

jobRunId - The job run id for the finished job run.

Returns:

true if the caller was the first to finish the job, false if another caller finished the job in the meantime.

Throws:

RunStorageException - error

deleteJobRun

void deleteJobRun(java.lang.String jobName,
                java.lang.String jobRunId)
                  throws RunStorageException

Removes the job run data of the given job run.

Parameters:

jobName - the job

jobRunId - the job run for which to remove the data

Throws:

RunStorageException - error

cancelJobRun

java.util.List<java.lang.String> cancelJobRun(java.lang.String jobName,
                                            java.lang.String jobRunId)
                                              throws RunStorageException

Cancels the given job run data -> active workflow runs and current tasks will be canceled. (Does not set job run state!).

Parameters:

jobName - the job

jobRunId - the job run for which to remove the data

Returns:

the ids of the workflow runs that were canceled.

Throws:

RunStorageException - error

getCurrentJobs

java.util.Collection<java.lang.String> getCurrentJobs()
                                                      throws RunStorageException

Returns:

names of job that have currently run data stored in this RunStorage. Usually it means that they are JobState.PREPARING,JobState.RUNNING, or JobState.FINISHING, but while cleaning up jobs in states JobState.SUCCEEDED and JobState.FAILED, they are still in RunStorage, too.

Throws:

RunStorageException - error

getJobRunId

java.lang.String getJobRunId(java.lang.String jobName)
                             throws RunStorageException

Parameters:

jobName - the job

Returns:

the id of the currently active job run, or 'null' if no job run is active.

Throws:

RunStorageException - error

getJobRunData

AnyMap getJobRunData(java.lang.String jobName,
                   boolean returnDetails)
                     throws RunStorageException

Parameters:

jobName - job

returnDetails - whether to return details (e.g. workflow/bucket definition)

Returns:

the data for the currently active job run, or 'null' if no job run is active.

Throws:

RunStorageException - error

getWorkflowRunData

AnyMap getWorkflowRunData(java.lang.String jobName,
                        java.lang.String workflowRunId)
                          throws RunStorageException

get information about a workflow run. The result contains these keys with integer values:

• JobManagerConstants.DATA_WORKFLOW_RUN_NO_OF_ACTIVE_TASKS
• JobManagerConstants.DATA_WORKFLOW_RUN_NO_OF_TRANSIENT_BULKS

The result may contain additional keys.

Returns:

workflow run information

Throws:

RunStorageException - jobrun or workflowrun are not active anymore.

setJobState

void setJobState(java.lang.String jobName,
               java.lang.String jobRunId,
               JobState jobState)
                 throws RunStorageException

Sets the state for the given job run. Do not invoke concurrently! This method does not check any preconditions and overwrites the state without further notice. Please check that you are the only one to set the state for a job run before invoking this method!

Parameters:

jobName - job

jobRunId - job run

jobState - the new state to set

Throws:

RunStorageException - error

setJobState

boolean setJobState(java.lang.String jobName,
                  java.lang.String jobRunId,
                  JobState expectedState,
                  JobState newState)
                    throws RunStorageException

Sets the state for the given job run if the given expected state matches the current state.

Parameters:

jobName - job

jobRunId - job run

expectedState - the old state that is expected to be the current state

newState - the new state to set

Returns:

'true', if new state could be set, otherwise 'false'.

Throws:

RunStorageException - error

getJobState

JobState getJobState(java.lang.String jobName)
                     throws RunStorageException

Parameters:

jobName - job

Returns:

the state of the currently active job run for the given job, or 'null' if no job run is active.

Throws:

RunStorageException - error

getJobState

JobState getJobState(java.lang.String jobName,
                   java.lang.String jobRunId)
                     throws RunStorageException

Parameters:

jobName - job

jobRunId - job run

Returns:

the state of the given job run for the given job, or 'null' if this job run is not active

Throws:

RunStorageException - error

getJobRunInfo

JobRunInfo getJobRunInfo(java.lang.String jobName)
                         throws RunStorageException

Parameters:

jobName - job

Returns:

if job is defined and active: id and state of current run. Else null.

Throws:

RunStorageException - error

startWorkflowRun

java.lang.String startWorkflowRun(java.lang.String jobName,
                                java.lang.String jobRunId)
                                  throws RunStorageException

Called when a new workflow run is started for the given job run.

Parameters:

jobName - job

jobRunId - job run

Returns:

the workflow run id of the new workflow run.

Throws:

RunStorageException - error

startCompletionWorkflowRun

java.lang.String startCompletionWorkflowRun(java.lang.String jobName,
                                          java.lang.String jobRunId)
                                            throws RunStorageException

Called when a new completing workflow run is started for the given job run.

Parameters:

jobName - job

jobRunId - job run

Returns:

the workflow run id of the new workflow run.

Throws:

RunStorageException - error

prepareToFinishWorkflowRun

boolean prepareToFinishWorkflowRun(java.lang.String jobName,
                                 java.lang.String jobRunId,
                                 java.lang.String workflowRunId)
                                   throws RunStorageException

Called before a workflow run is finished and deleted. Decides whether this thread will be allowed to finish the workflow run. Two flags are set here to avoid concurrent races: 1.) a temporary job run data map entry 'workflow-run-finish-' is set to a locally created id, to make sure that only one thread is doing the finishing. 2.) a temporary ephemeral 'finishing' flag is set to show that this workflow run is currently finished. This is needed because server could be crashed in between. If entry above is set and flag is missing, than entry above can be overwritten! Both entries are removed in deleteWorkflowRun(String, String, String).

Returns:

'true' if the calling thread is the one to finish the workflow, false if not.

Throws:

RunStorageException

successfulWorkflowRun

void successfulWorkflowRun(java.lang.String jobName,
                         java.lang.String jobRunId,
                         java.lang.String workflowRunId)
                           throws RunStorageException

Called when a workflow run successfully finished. The workflow run data is merged into the job run data. Does not delete the data of the workflow run.

Throws:

RunStorageException

failedWorkflowRun

void failedWorkflowRun(java.lang.String jobName,
                     java.lang.String jobRunId,
                     java.lang.String workflowRunId)
                       throws RunStorageException

Called when a workflow run failed. The workflow run data is merged into the job run data. Does not delete the data of the workflow run.

Throws:

RunStorageException

deleteWorkflowRunData

void deleteWorkflowRunData(java.lang.String jobName,
                         java.lang.String jobRunId,
                         java.lang.String workflowRunId)
                           throws RunStorageException

Deletes the data of the given workflow run.

Throws:

RunStorageException

deleteWorkflowRun

void deleteWorkflowRun(java.lang.String jobName,
                     java.lang.String jobRunId,
                     java.lang.String workflowRunId)
                       throws RunStorageException

Deletes the given workflow run.

Throws:

RunStorageException

checkAndCleanupActiveWorkflowRuns

boolean checkAndCleanupActiveWorkflowRuns(java.lang.String jobName,
                                        java.lang.String jobRunId)
                                          throws RunStorageException

Checks if there are active workflow runs. Also checks if there are completed workflow runs that someone else started finishing but couldn't succeed. Such completed workflow runs are cleaned up. (finished + removed)

Returns:

'true' if there are active workflow runs, otherwise 'false'.

Throws:

RunStorageException

hasWorkflowRun

boolean hasWorkflowRun(java.lang.String jobName,
                     java.lang.String jobRunId,
                     java.lang.String workflowRunId)
                       throws RunStorageException

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

Returns:

'true' if the given workflow run exists.

Throws:

RunStorageException - error

hasTasks

boolean hasTasks(java.lang.String jobName,
               java.lang.String jobRunId,
               java.lang.String workflowRunId)
                 throws RunStorageException

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

Returns:

'true' if there are active tasks in the given workflow run, otherwise 'false'

Throws:

RunStorageException - error

startTask

void startTask(java.lang.String jobName,
             java.lang.String jobRunId,
             java.lang.String workflowRunId,
             java.lang.String stepId,
             java.lang.String taskId)
               throws RunStorageException

Called when a new task is started for the given step in the given workflow of the given job run.

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

stepId - step id

taskId - task id

Throws:

RunStorageException - if task already exists, or other error while storing task

startTask

void startTask(java.lang.String jobName,
             java.lang.String jobRunId,
             java.lang.String workflowRunId,
             java.lang.String stepId,
             java.lang.String taskId,
             int numberOfRetries)
               throws RunStorageException

Called when a new task is started for the given step in the given workflow of the given job run. The numberOfRetries parameter adds the possibility to store the number of times this task has already been restarted.

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

stepId - step id

taskId - task id

numberOfRetries - current number of retries for this task

Throws:

RunStorageException - if task already exists, or other error while storing task

startTasks

void startTasks(java.lang.String jobName,
              java.lang.String jobRunId,
              java.lang.String workflowRunId,
              java.util.Collection<Task> tasks)
                throws RunStorageException

Start multiple new tasks for a given job/workflow run at once.

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

tasks - new tasks

Throws:

RunStorageException - error while storing task

getTaskRetries

int getTaskRetries(java.lang.String jobName,
                 java.lang.String jobRunId,
                 java.lang.String workflowRunId,
                 java.lang.String stepId,
                 java.lang.String taskId)
                   throws RunStorageException

get the number of retries for the task.

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

stepId - step id

taskId - task id

Returns:

The number of retries for the task.

Throws:

RunStorageException - if task already exists, or other error while storing task

finishTask

void finishTask(java.lang.String jobName,
              java.lang.String jobRunId,
              java.lang.String workflowRunId,
              java.lang.String stepId,
              java.lang.String taskId,
              java.util.Map<java.lang.String,java.lang.Number> workerCounter,
              java.util.Map<java.lang.String,java.lang.String> properties)
                throws RunStorageException

Called when a task is successfully finished. Deletes the data of a given task for the given step in the given workflow of the given job run.

Throws:

RunStorageException

obsoleteTask

void obsoleteTask(java.lang.String jobName,
                java.lang.String jobRunId,
                java.lang.String workflowRunId,
                java.lang.String stepId,
                java.lang.String taskId,
                java.util.Map<java.lang.String,java.lang.String> properties)
                  throws RunStorageException

Called when a task finished as obsolete. Deletes the data of a given task for the given step in the given workflow of the given job run.

Throws:

RunStorageException

retriedTask

void retriedTask(java.lang.String jobName,
               java.lang.String jobRunId,
               java.lang.String workflowRunId,
               java.lang.String stepId,
               java.lang.String taskId,
               boolean retryByWorker,
               java.util.Map<java.lang.String,java.lang.String> properties)
                 throws RunStorageException

Called when a task is retried. Deletes the data of a given task for the given step in the given workflow of the given job run.

Parameters:

retryByWorker - whether the retry was triggered by a worker

Throws:

RunStorageException

failedTask

void failedTask(java.lang.String jobName,
              java.lang.String jobRunId,
              java.lang.String workflowRunId,
              java.lang.String stepId,
              java.lang.String taskId,
              boolean failedAfterRetry,
              java.util.Map<java.lang.String,java.lang.String> properties)
                throws RunStorageException

Called when a task is failed. Deletes the data of a given task for the given step in the given workflow of the given job run.

Throws:

RunStorageException

addTransientBulk

void addTransientBulk(java.lang.String jobName,
                    java.lang.String jobRunId,
                    java.lang.String workflowRunId,
                    java.lang.String bulk)
                      throws RunStorageException

Stores a transient bulk for the given workflow run.

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

bulk - the bulk id

Throws:

RunStorageException - error

getTransientBulks

java.util.Collection<java.lang.String> getTransientBulks(java.lang.String jobName,
                                                       java.lang.String jobRunId,
                                                       java.lang.String workflowRunId)
                                                         throws RunStorageException

Parameters:

jobName - job

jobRunId - job run

workflowRunId - workflow run

Returns:

the transient bulk ids for the given workflow run

Throws:

RunStorageException - error

getTriggeredJobs

java.util.Collection<java.lang.String> getTriggeredJobs(java.lang.String bucketId)
                                                        throws RunStorageException

Parameters:

bucketId - a bucket ID

Returns:

names of jobs triggered by this bucket

Throws:

RunStorageException - error

addJobTrigger

void addJobTrigger(java.lang.String bucketId,
                 java.lang.String jobName)
                   throws RunStorageException

Parameters:

bucketId - a bucket ID

jobName - job name

Throws:

RunStorageException - error

removeJobTrigger

void removeJobTrigger(java.lang.String bucketId,
                    java.lang.String jobName)
                      throws RunStorageException

Parameters:

bucketId - a bucket ID

jobName - job name

Throws:

RunStorageException - error

hasTask

boolean hasTask(java.lang.String jobName,
              java.lang.String workflowRunId,
              java.lang.String taskId)
                throws RunStorageException

Check if the given task with the given task id exists for the job and workflowrun.

Parameters:

jobName - the name of the job.

workflowRunId - the id of the workflow run

taskId - the id of the task

Returns:

'true' if such a task exists, 'false' if not.

Throws:

RunStorageException - an exception occurred while checking for task existence.

getCurrentTimestamp

java.lang.String getCurrentTimestamp()

Returns:

string with current timestamp

registerJobRunListener

void registerJobRunListener(JobRunListener jobRunListener,
                          java.lang.String jobName)

Parameters:

jobRunListener - listener to be notified about job run events.

jobName - job on which to listen

setCustomData

void setCustomData(java.lang.String jobName,
                 java.lang.String worker,
                 byte[] data,
                 java.lang.String... path)
                   throws RunStorageException

Store custom data for given job, worker and path. If not needed, 'path' parameter can be left out. If path already exists, existing data will be overwritten.

Throws:

RunStorageException - if job does not exist in RunStorage.

getCustomData

byte[] getCustomData(java.lang.String jobName,
                   java.lang.String worker,
                   java.lang.String... path)
                     throws RunStorageException

Returns:

custom data for given job (run), worker and path. Returns 'null' if no data is stored or path constructed from parameters doesn't exist.

Throws:

RunStorageException

setupBarrier

void setupBarrier(java.lang.String jobName,
                java.lang.String workflowRunId,
                int position)
                  throws RunStorageException

Throws:

RunStorageException

addTaskForBarrier

void addTaskForBarrier(java.lang.String jobName,
                     java.lang.String workflowRunId,
                     int position,
                     java.lang.String taskId)
                       throws RunStorageException

Throws:

RunStorageException

removeTaskForBarrier

void removeTaskForBarrier(java.lang.String jobName,
                        java.lang.String workflowRunId,
                        int position,
                        java.lang.String taskId)
                          throws RunStorageException

Throws:

RunStorageException

tasksLockBarrier

boolean tasksLockBarrier(java.lang.String jobName,
                       java.lang.String workflowRunId,
                       int position)
                         throws RunStorageException

Throws:

RunStorageException

addBulkForBarrier

void addBulkForBarrier(java.lang.String jobName,
                     java.lang.String workflowRunId,
                     int position,
                     java.lang.String bulkId)
                       throws RunStorageException

Throws:

RunStorageException

getBulksForBarrier

java.util.Collection<java.lang.String> getBulksForBarrier(java.lang.String jobName,
                                                        java.lang.String workflowRunId,
                                                        int position)
                                                          throws RunStorageException

Throws:

RunStorageException

cleanOpenedBarrier

void cleanOpenedBarrier(java.lang.String jobName,
                      java.lang.String workflowRunId,
                      int position)
                        throws RunStorageException

Throws:

RunStorageException

prepareOpeningOfBarrier

boolean prepareOpeningOfBarrier(java.lang.String jobName,
                              java.lang.String workflowRunId,
                              int position)
                                throws RunStorageException

Throws:

RunStorageException