240 lines
7.6 KiB
Lua
240 lines
7.6 KiB
Lua
--[[
|
|
Move job from active to a finished status (completed or failed)
|
|
A job can only be moved to completed if it was active.
|
|
The job must be locked before it can be moved to a finished status,
|
|
and the lock must be released in this script.
|
|
|
|
Input:
|
|
KEYS[1] wait key
|
|
KEYS[2] active key
|
|
KEYS[3] prioritized key
|
|
KEYS[4] event stream key
|
|
KEYS[5] stalled key
|
|
|
|
-- Rate limiting
|
|
KEYS[6] rate limiter key
|
|
KEYS[7] delayed key
|
|
|
|
KEYS[8] paused key
|
|
KEYS[9] meta key
|
|
KEYS[10] pc priority counter
|
|
|
|
KEYS[11] completed/failed key
|
|
KEYS[12] jobId key
|
|
KEYS[13] metrics key
|
|
KEYS[14] marker key
|
|
|
|
ARGV[1] jobId
|
|
ARGV[2] timestamp
|
|
ARGV[3] msg property returnvalue / failedReason
|
|
ARGV[4] return value / failed reason
|
|
ARGV[5] target (completed/failed)
|
|
ARGV[6] fetch next?
|
|
ARGV[7] keys prefix
|
|
ARGV[8] opts
|
|
ARGV[9] job fields to update
|
|
|
|
opts - token - lock token
|
|
opts - keepJobs
|
|
opts - lockDuration - lock duration in milliseconds
|
|
opts - attempts max attempts
|
|
opts - maxMetricsSize
|
|
opts - fpof - fail parent on fail
|
|
opts - cpof - continue parent on fail
|
|
opts - idof - ignore dependency on fail
|
|
opts - rdof - remove dependency on fail
|
|
opts - name - worker name
|
|
|
|
Output:
|
|
0 OK
|
|
-1 Missing key.
|
|
-2 Missing lock.
|
|
-3 Job not in active set
|
|
-4 Job has pending children
|
|
-6 Lock is not owned by this client
|
|
-9 Job has failed children
|
|
|
|
Events:
|
|
'completed/failed'
|
|
]]
|
|
local rcall = redis.call
|
|
|
|
--- Includes
|
|
--- @include "includes/collectMetrics"
|
|
--- @include "includes/fetchNextJob"
|
|
--- @include "includes/moveChildFromDependenciesIfNeeded"
|
|
--- @include "includes/removeDeduplicationKeyIfNeededOnFinalization"
|
|
--- @include "includes/removeJobKeys"
|
|
--- @include "includes/removeJobsByMaxAge"
|
|
--- @include "includes/removeJobsByMaxCount"
|
|
--- @include "includes/removeLock"
|
|
--- @include "includes/removeParentDependencyKey"
|
|
--- @include "includes/requeueDeduplicatedJob"
|
|
--- @include "includes/trimEvents"
|
|
--- @include "includes/updateParentDepsIfNeeded"
|
|
--- @include "includes/updateJobFields"
|
|
|
|
local jobIdKey = KEYS[12]
|
|
if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
-- Make sure it does not have pending dependencies
|
|
-- It must happen before removing lock
|
|
if ARGV[5] == "completed" then
|
|
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
|
|
return -4
|
|
end
|
|
|
|
if rcall("ZCARD", jobIdKey .. ":unsuccessful") ~= 0 then
|
|
return -9
|
|
end
|
|
end
|
|
|
|
local opts = cmsgpack.unpack(ARGV[8])
|
|
|
|
local token = opts['token']
|
|
|
|
local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
|
|
if errorCode < 0 then
|
|
return errorCode
|
|
end
|
|
|
|
updateJobFields(jobIdKey, ARGV[9]);
|
|
|
|
local attempts = opts['attempts']
|
|
local maxMetricsSize = opts['maxMetricsSize']
|
|
local maxCount = opts['keepJobs']['count']
|
|
local maxAge = opts['keepJobs']['age']
|
|
local maxLimit = opts['keepJobs']['limit'] or 1000
|
|
|
|
local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
|
|
local parentKey = jobAttributes[1] or ""
|
|
local parentId = ""
|
|
local parentQueueKey = ""
|
|
if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed
|
|
local jsonDecodedParent = cjson.decode(jobAttributes[2])
|
|
parentId = jsonDecodedParent['id']
|
|
parentQueueKey = jsonDecodedParent['queueKey']
|
|
end
|
|
|
|
local jobId = ARGV[1]
|
|
local timestamp = ARGV[2]
|
|
|
|
-- Remove from active list (if not active we shall return error)
|
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
|
|
|
if (numRemovedElements < 1) then
|
|
return -3
|
|
end
|
|
|
|
local eventStreamKey = KEYS[4]
|
|
local metaKey = KEYS[9]
|
|
-- Trim events before emitting them to avoid trimming events emitted in this script
|
|
trimEvents(metaKey, eventStreamKey)
|
|
|
|
local prefix = ARGV[7]
|
|
|
|
removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
|
|
|
|
-- Check if there is requeue data for this dedup ID (keepLastIfActive mode)
|
|
if jobAttributes[3] then
|
|
requeueDeduplicatedJob(prefix, jobAttributes[3], eventStreamKey,
|
|
metaKey, KEYS[2], KEYS[1], KEYS[8], KEYS[14], KEYS[3], KEYS[10],
|
|
KEYS[7], timestamp)
|
|
end
|
|
|
|
-- If job has a parent we need to
|
|
-- 1) remove this job id from parents dependencies
|
|
-- 2) move the job Id to parent "processed" set
|
|
-- 3) push the results into parent "results" list
|
|
-- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
|
|
if parentId == "" and parentKey ~= "" then
|
|
parentId = getJobIdFromKey(parentKey)
|
|
parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
|
|
end
|
|
|
|
if parentId ~= "" then
|
|
if ARGV[5] == "completed" then
|
|
local dependenciesSet = parentKey .. ":dependencies"
|
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
|
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
|
|
timestamp)
|
|
end
|
|
else
|
|
moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp)
|
|
end
|
|
end
|
|
|
|
local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
|
|
|
|
-- Remove job?
|
|
if maxCount ~= 0 then
|
|
local targetSet = KEYS[11]
|
|
-- Add to complete/failed set
|
|
rcall("ZADD", targetSet, timestamp, jobId)
|
|
rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
|
|
|
if ARGV[5] == "failed" then
|
|
rcall("HDEL", jobIdKey, "defa")
|
|
end
|
|
|
|
-- Remove old jobs?
|
|
if maxAge ~= nil then
|
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
|
|
end
|
|
|
|
if maxCount ~= nil and maxCount > 0 then
|
|
removeJobsByMaxCount(maxCount, targetSet, prefix)
|
|
end
|
|
else
|
|
removeJobKeys(jobIdKey)
|
|
if parentKey ~= "" then
|
|
-- TODO: when a child is removed when finished, result or failure in parent
|
|
-- must not be deleted, those value references should be deleted when the parent
|
|
-- is deleted
|
|
removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
|
|
end
|
|
end
|
|
|
|
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
|
|
|
|
if ARGV[5] == "failed" then
|
|
if tonumber(attemptsMade) >= tonumber(attempts) then
|
|
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
|
|
attemptsMade)
|
|
end
|
|
end
|
|
|
|
-- Collect metrics
|
|
if maxMetricsSize ~= "" then
|
|
collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
|
|
end
|
|
|
|
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
|
|
-- and not rate limited.
|
|
if (ARGV[6] == "1") then
|
|
local result = fetchNextJob(KEYS[1], KEYS[2], KEYS[3], eventStreamKey,
|
|
KEYS[6], KEYS[7], KEYS[8], metaKey, KEYS[10], KEYS[14], prefix,
|
|
timestamp, opts)
|
|
if result then
|
|
return result
|
|
end
|
|
end
|
|
|
|
local waitLen = rcall("LLEN", KEYS[1])
|
|
if waitLen == 0 then
|
|
local activeLen = rcall("LLEN", KEYS[2])
|
|
|
|
if activeLen == 0 then
|
|
local prioritizedLen = rcall("ZCARD", KEYS[3])
|
|
|
|
if prioritizedLen == 0 then
|
|
rcall("XADD", eventStreamKey, "*", "event", "drained")
|
|
end
|
|
end
|
|
end
|
|
|
|
return 0
|
|
else
|
|
return -1
|
|
end
|