1040 lines
39 KiB
JavaScript
1040 lines
39 KiB
JavaScript
const content = `--[[
|
|
Move job from active to a finished status (completed or failed)
|
|
A job can only be moved to completed if it was active.
|
|
The job must be locked before it can be moved to a finished status,
|
|
and the lock must be released in this script.
|
|
Input:
|
|
KEYS[1] wait key
|
|
KEYS[2] active key
|
|
KEYS[3] prioritized key
|
|
KEYS[4] event stream key
|
|
KEYS[5] stalled key
|
|
-- Rate limiting
|
|
KEYS[6] rate limiter key
|
|
KEYS[7] delayed key
|
|
KEYS[8] paused key
|
|
KEYS[9] meta key
|
|
KEYS[10] pc priority counter
|
|
KEYS[11] completed/failed key
|
|
KEYS[12] jobId key
|
|
KEYS[13] metrics key
|
|
KEYS[14] marker key
|
|
ARGV[1] jobId
|
|
ARGV[2] timestamp
|
|
ARGV[3] msg property returnvalue / failedReason
|
|
ARGV[4] return value / failed reason
|
|
ARGV[5] target (completed/failed)
|
|
ARGV[6] fetch next?
|
|
ARGV[7] keys prefix
|
|
ARGV[8] opts
|
|
ARGV[9] job fields to update
|
|
opts - token - lock token
|
|
opts - keepJobs
|
|
opts - lockDuration - lock duration in milliseconds
|
|
opts - attempts max attempts
|
|
opts - maxMetricsSize
|
|
opts - fpof - fail parent on fail
|
|
opts - cpof - continue parent on fail
|
|
opts - idof - ignore dependency on fail
|
|
opts - rdof - remove dependency on fail
|
|
opts - name - worker name
|
|
Output:
|
|
0 OK
|
|
-1 Missing key.
|
|
-2 Missing lock.
|
|
-3 Job not in active set
|
|
-4 Job has pending children
|
|
-6 Lock is not owned by this client
|
|
-9 Job has failed children
|
|
Events:
|
|
'completed/failed'
|
|
]]
|
|
local rcall = redis.call
|
|
--- Includes
|
|
--[[
|
|
Functions to collect metrics based on a current and previous count of jobs.
|
|
Granularity is fixed at 1 minute.
|
|
]]
|
|
--[[
|
|
Function to loop in batches.
|
|
Just a bit of warning, some commands as ZREM
|
|
could receive a maximum of 7000 parameters per call.
|
|
]]
|
|
local function batches(n, batchSize)
|
|
local i = 0
|
|
return function()
|
|
local from = i * batchSize + 1
|
|
i = i + 1
|
|
if (from <= n) then
|
|
local to = math.min(from + batchSize - 1, n)
|
|
return from, to
|
|
end
|
|
end
|
|
end
|
|
local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
|
|
timestamp)
|
|
-- Increment current count
|
|
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
|
|
-- Compute how many data points we need to add to the list, N.
|
|
local prevTS = rcall("HGET", metaKey, "prevTS")
|
|
if not prevTS then
|
|
-- If prevTS is nil, set it to the current timestamp
|
|
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
|
|
return
|
|
end
|
|
local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints))
|
|
if N > 0 then
|
|
local delta = count - rcall("HGET", metaKey, "prevCount")
|
|
-- If N > 1, add N-1 zeros to the list
|
|
if N > 1 then
|
|
local points = {}
|
|
points[1] = delta
|
|
for i = 2, N do
|
|
points[i] = 0
|
|
end
|
|
for from, to in batches(#points, 7000) do
|
|
rcall("LPUSH", dataPointsList, unpack(points, from, to))
|
|
end
|
|
else
|
|
-- LPUSH delta to the list
|
|
rcall("LPUSH", dataPointsList, delta)
|
|
end
|
|
-- LTRIM to keep list to its max size
|
|
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
|
|
-- update prev count with current count
|
|
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
|
|
end
|
|
end
|
|
--[[
|
|
Function to fetch the next job to process.
|
|
Tries to get the next job to avoid an extra roundtrip if the queue is
|
|
not closing and not rate limited.
|
|
Input:
|
|
waitKey - wait list key
|
|
activeKey - active list key
|
|
prioritizedKey - prioritized sorted set key
|
|
eventStreamKey - event stream key
|
|
rateLimiterKey - rate limiter key
|
|
delayedKey - delayed sorted set key
|
|
pausedKey - paused list key
|
|
metaKey - meta hash key
|
|
pcKey - priority counter key
|
|
markerKey - marker key
|
|
prefix - keys prefix
|
|
timestamp - current timestamp
|
|
opts - options table:
|
|
token (required) - lock token used when locking jobs
|
|
lockDuration (required) - lock duration for acquired jobs
|
|
limiter (optional) - rate limiter options table (e.g. { max = number })
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to get queue metadata.
|
|
]]
|
|
local function getQueueMetadata(queueMetaKey, activeKey, waitKey)
|
|
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
if queueAttributes[1] then
|
|
return true, queueAttributes[3], queueAttributes[4]
|
|
else
|
|
if queueAttributes[2] then
|
|
local activeCount = rcall("LLEN", activeKey)
|
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
return true, queueAttributes[3], queueAttributes[4]
|
|
else
|
|
return false, queueAttributes[3], queueAttributes[4]
|
|
end
|
|
end
|
|
end
|
|
return false, queueAttributes[3], queueAttributes[4]
|
|
end
|
|
--[[
|
|
Function to return the next delayed job timestamp.
|
|
]]
|
|
local function getNextDelayedTimestamp(delayedKey)
|
|
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
|
if #result then
|
|
local nextTimestamp = tonumber(result[2])
|
|
if nextTimestamp ~= nil then
|
|
return nextTimestamp / 0x1000
|
|
end
|
|
end
|
|
end
|
|
--[[
|
|
Function to get current rate limit ttl.
|
|
]]
|
|
local function getRateLimitTTL(maxJobs, rateLimiterKey)
|
|
if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
|
|
local pttl = rcall("PTTL", rateLimiterKey)
|
|
if pttl == 0 then
|
|
rcall("DEL", rateLimiterKey)
|
|
end
|
|
if pttl > 0 then
|
|
return pttl
|
|
end
|
|
end
|
|
return 0
|
|
end
|
|
--[[
|
|
Function to move job from prioritized state to active.
|
|
]]
|
|
local function moveJobFromPrioritizedToActive(priorityKey, activeKey, priorityCounterKey)
|
|
local prioritizedJob = rcall("ZPOPMIN", priorityKey)
|
|
if #prioritizedJob > 0 then
|
|
rcall("LPUSH", activeKey, prioritizedJob[1])
|
|
return prioritizedJob[1]
|
|
else
|
|
rcall("DEL", priorityCounterKey)
|
|
end
|
|
end
|
|
--[[
|
|
Function to move job from wait state to active.
|
|
Input:
|
|
opts - token - lock token
|
|
opts - lockDuration
|
|
opts - limiter
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Add marker if needed when a job is available.
|
|
]]
|
|
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
if not isPausedOrMaxed then
|
|
rcall("ZADD", markerKey, 0, "0")
|
|
end
|
|
end
|
|
local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
|
|
jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
|
|
local jobKey = keyPrefix .. jobId
|
|
-- Check if we need to perform rate limiting.
|
|
if maxJobs then
|
|
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
|
if jobCounter == 1 then
|
|
local integerDuration = math.floor(math.abs(limiterDuration))
|
|
rcall("PEXPIRE", rateLimiterKey, integerDuration)
|
|
end
|
|
end
|
|
-- get a lock
|
|
if opts['token'] ~= "0" then
|
|
local lockKey = jobKey .. ':lock'
|
|
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
|
end
|
|
local optionalValues = {}
|
|
if opts['name'] then
|
|
-- Set "processedBy" field to the worker name
|
|
table.insert(optionalValues, "pb")
|
|
table.insert(optionalValues, opts['name'])
|
|
end
|
|
rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
|
rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
|
|
rcall("HINCRBY", jobKey, "ats", 1)
|
|
addBaseMarkerIfNeeded(markerKey, false)
|
|
-- rate limit delay must be 0 in this case to prevent adding more delay
|
|
-- when job that is moved to active needs to be processed
|
|
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
|
end
|
|
--[[
|
|
Updates the delay set, by moving delayed jobs that should
|
|
be processed now to "wait".
|
|
Events:
|
|
'waiting'
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to add job in target list and add marker if needed.
|
|
]]
|
|
-- Includes
|
|
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
|
rcall(pushCmd, targetKey, jobId)
|
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
end
|
|
--[[
|
|
Function to add job considering priority.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to get priority score.
|
|
]]
|
|
local function getPriorityScore(priority, priorityCounterKey)
|
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
|
return priority * 0x100000000 + prioCounter % 0x100000000
|
|
end
|
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
|
isPausedOrMaxed)
|
|
local score = getPriorityScore(priority, priorityCounterKey)
|
|
rcall("ZADD", prioritizedKey, score, jobId)
|
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
end
|
|
-- Try to get as much as 1000 jobs at once
|
|
local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
|
|
eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
|
|
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000 - 1, "LIMIT", 0, 1000)
|
|
if (#jobs > 0) then
|
|
rcall("ZREM", delayedKey, unpack(jobs))
|
|
for _, jobId in ipairs(jobs) do
|
|
local jobKey = prefix .. jobId
|
|
local priority =
|
|
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
|
if priority == 0 then
|
|
-- LIFO or FIFO
|
|
rcall("LPUSH", targetKey, jobId)
|
|
else
|
|
local score = getPriorityScore(priority, priorityCounterKey)
|
|
rcall("ZADD", prioritizedKey, score, jobId)
|
|
end
|
|
-- Emit waiting event
|
|
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
|
jobId, "prev", "delayed")
|
|
rcall("HSET", jobKey, "delay", 0)
|
|
end
|
|
addBaseMarkerIfNeeded(markerKey, isPaused)
|
|
end
|
|
end
|
|
local function fetchNextJob(waitKey, activeKey, prioritizedKey, eventStreamKey,
|
|
rateLimiterKey, delayedKey, pausedKey, metaKey, pcKey, markerKey, prefix,
|
|
timestamp, opts)
|
|
local isPausedOrMaxed, rateLimitMax, rateLimitDuration =
|
|
getQueueMetadata(metaKey, activeKey, waitKey)
|
|
-- Check if there are delayed jobs that can be promoted
|
|
promoteDelayedJobs(delayedKey, markerKey, waitKey, prioritizedKey,
|
|
eventStreamKey, prefix, timestamp, pcKey, isPausedOrMaxed)
|
|
local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
|
|
-- Check if we are rate limited first.
|
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
|
if expireTime > 0 then
|
|
return {0, 0, expireTime, 0}
|
|
end
|
|
-- paused or maxed queue
|
|
if isPausedOrMaxed then
|
|
return {0, 0, 0, 0}
|
|
end
|
|
local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
|
|
local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
|
if jobId then
|
|
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
|
if string.sub(jobId, 1, 2) == "0:" then
|
|
rcall("LREM", activeKey, 1, jobId)
|
|
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
|
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
|
if jobId == "0:0" then
|
|
jobId = moveJobFromPrioritizedToActive(prioritizedKey, activeKey, pcKey)
|
|
return prepareJobForProcessing(prefix, rateLimiterKey,
|
|
eventStreamKey, jobId, timestamp, maxJobs,
|
|
limiterDuration, markerKey, opts)
|
|
end
|
|
else
|
|
return prepareJobForProcessing(prefix, rateLimiterKey,
|
|
eventStreamKey, jobId, timestamp, maxJobs,
|
|
limiterDuration, markerKey, opts)
|
|
end
|
|
else
|
|
jobId = moveJobFromPrioritizedToActive(prioritizedKey, activeKey, pcKey)
|
|
if jobId then
|
|
return prepareJobForProcessing(prefix, rateLimiterKey,
|
|
eventStreamKey, jobId, timestamp, maxJobs,
|
|
limiterDuration, markerKey, opts)
|
|
end
|
|
end
|
|
-- Return the timestamp for the next delayed job if any.
|
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
|
if nextTimestamp ~= nil then
|
|
-- The result is guaranteed to be positive, since the
|
|
-- ZRANGEBYSCORE command would have return a job otherwise.
|
|
return {0, 0, 0, nextTimestamp}
|
|
end
|
|
end
|
|
--[[
|
|
Function to recursively move from waitingChildren to failed.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
|
if no pending dependencies.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Move parent to a wait status (wait, prioritized or delayed)
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Add delay marker if needed.
|
|
]]
|
|
-- Includes
|
|
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
|
if nextTimestamp ~= nil then
|
|
-- Replace the score of the marker with the newest known
|
|
-- next timestamp.
|
|
rcall("ZADD", markerKey, nextTimestamp, "1")
|
|
end
|
|
end
|
|
--[[
|
|
Function to check if queue is paused or maxed
|
|
(since an empty list and !EXISTS are not really the same).
|
|
]]
|
|
local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
if queueAttributes[1] then
|
|
return true
|
|
else
|
|
if queueAttributes[2] then
|
|
local activeCount = rcall("LLEN", activeKey)
|
|
return activeCount >= tonumber(queueAttributes[2])
|
|
end
|
|
end
|
|
return false
|
|
end
|
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
local parentWaitKey = parentQueueKey .. ":wait"
|
|
local parentActiveKey = parentQueueKey .. ":active"
|
|
local parentMetaKey = parentQueueKey .. ":meta"
|
|
local parentMarkerKey = parentQueueKey .. ":marker"
|
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
|
local priority = tonumber(jobAttributes[1]) or 0
|
|
local delay = tonumber(jobAttributes[2]) or 0
|
|
if delay > 0 then
|
|
local delayedTimestamp = tonumber(timestamp) + delay
|
|
local score = delayedTimestamp * 0x1000
|
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
|
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
|
delayedTimestamp)
|
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
|
else
|
|
if priority == 0 then
|
|
local isParentPausedOrMaxed =
|
|
isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
|
addJobInTargetList(parentWaitKey, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
|
else
|
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
|
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
|
parentQueueKey .. ":pc", isPausedOrMaxed)
|
|
end
|
|
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
|
"waiting-children")
|
|
end
|
|
end
|
|
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
|
if rcall("EXISTS", parentKey) == 1 then
|
|
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
|
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
|
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
|
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
end
|
|
end
|
|
end
|
|
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
|
parentId, timestamp)
|
|
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
|
if doNotHavePendingDependencies then
|
|
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
|
end
|
|
end
|
|
local handleChildFailureAndMoveParentToWait = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
|
if rcall("EXISTS", parentKey) == 1 then
|
|
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
|
local parentWaitingChildrenOrDelayedKey
|
|
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
|
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
|
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
|
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
|
rcall("HSET", parentKey, "delay", 0)
|
|
end
|
|
if parentWaitingChildrenOrDelayedKey then
|
|
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
|
local deferredFailure = "child " .. jobIdKey .. " failed"
|
|
rcall("HSET", parentKey, "defa", deferredFailure)
|
|
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
else
|
|
if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
|
|
local deferredFailure = "child " .. jobIdKey .. " failed"
|
|
rcall("HSET", parentKey, "defa", deferredFailure)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
local moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
|
|
if rawParentData then
|
|
local parentData = cjson.decode(rawParentData)
|
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
|
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
|
if parentData['fpof'] then
|
|
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
|
local parentUnsuccessfulChildrenKey = parentKey .. ":unsuccessful"
|
|
rcall("ZADD", parentUnsuccessfulChildrenKey, timestamp, childKey)
|
|
handleChildFailureAndMoveParentToWait(
|
|
parentData['queueKey'],
|
|
parentKey,
|
|
parentData['id'],
|
|
childKey,
|
|
timestamp
|
|
)
|
|
end
|
|
elseif parentData['cpof'] then
|
|
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
|
local parentFailedChildrenKey = parentKey .. ":failed"
|
|
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
|
moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
|
end
|
|
elseif parentData['idof'] or parentData['rdof'] then
|
|
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
|
moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
|
|
parentKey, parentData['id'], timestamp)
|
|
if parentData['idof'] then
|
|
local parentFailedChildrenKey = parentKey .. ":failed"
|
|
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
--[[
|
|
Function to remove deduplication key if needed
|
|
when a job is moved to completed or failed states.
|
|
]]
|
|
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
|
deduplicationId, jobId)
|
|
if deduplicationId then
|
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
|
local pttl = rcall("PTTL", deduplicationKey)
|
|
if pttl == 0 then
|
|
return rcall("DEL", deduplicationKey)
|
|
end
|
|
if pttl == -1 then
|
|
local currentJobId = rcall('GET', deduplicationKey)
|
|
if currentJobId and currentJobId == jobId then
|
|
return rcall("DEL", deduplicationKey)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
--[[
|
|
Function to remove job keys.
|
|
]]
|
|
local function removeJobKeys(jobKey)
|
|
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
|
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
|
end
|
|
--[[
|
|
Functions to remove jobs by max age.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to remove job.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to remove deduplication key if needed
|
|
when a job is being removed.
|
|
]]
|
|
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
|
jobId, deduplicationId)
|
|
if deduplicationId then
|
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
|
local currentJobId = rcall('GET', deduplicationKey)
|
|
if currentJobId and currentJobId == jobId then
|
|
rcall("DEL", deduplicationKey)
|
|
-- Also clean up any pending dedup-next data for this dedup ID
|
|
rcall("DEL", prefixKey .. "dn:" .. deduplicationId)
|
|
return 1
|
|
end
|
|
end
|
|
end
|
|
--[[
|
|
Check if this job has a parent. If so we will just remove it from
|
|
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
|
which requires code from "moveToFinished"
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Functions to destructure job key.
|
|
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
|
]]
|
|
local getJobIdFromKey = function (jobKey)
|
|
return string.match(jobKey, ".*:(.*)")
|
|
end
|
|
local getJobKeyPrefix = function (jobKey, jobId)
|
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
|
end
|
|
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
local isPausedOrMaxed =
|
|
isQueuePausedOrMaxed(parentPrefix .. "meta", parentPrefix .. "active")
|
|
addJobInTargetList(parentPrefix .. "wait", parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
|
if emitEvent then
|
|
local parentEventStream = parentPrefix .. "events"
|
|
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
|
end
|
|
end
|
|
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
|
|
if parentKey then
|
|
local parentDependenciesKey = parentKey .. ":dependencies"
|
|
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
if result > 0 then
|
|
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
if pendingDependencies == 0 then
|
|
local parentId = getJobIdFromKey(parentKey)
|
|
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
|
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
if numRemovedElements == 1 then
|
|
if hard then -- remove parent in same queue
|
|
if parentPrefix == baseKey then
|
|
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
|
|
removeJobKeys(parentKey)
|
|
if debounceId then
|
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
|
end
|
|
else
|
|
_moveParentToWait(parentPrefix, parentId)
|
|
end
|
|
else
|
|
_moveParentToWait(parentPrefix, parentId, true)
|
|
end
|
|
end
|
|
end
|
|
return true
|
|
end
|
|
else
|
|
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
|
|
local missedParentKey = parentAttributes[1]
|
|
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
|
|
and (rcall("EXISTS", missedParentKey) == 1)) then
|
|
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
|
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
if result > 0 then
|
|
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
if pendingDependencies == 0 then
|
|
local parentId = getJobIdFromKey(missedParentKey)
|
|
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
|
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
if numRemovedElements == 1 then
|
|
if hard then
|
|
if parentPrefix == baseKey then
|
|
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
|
|
removeJobKeys(missedParentKey)
|
|
if parentAttributes[2] then
|
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
|
end
|
|
else
|
|
_moveParentToWait(parentPrefix, parentId)
|
|
end
|
|
else
|
|
_moveParentToWait(parentPrefix, parentId, true)
|
|
end
|
|
end
|
|
end
|
|
return true
|
|
end
|
|
end
|
|
end
|
|
return false
|
|
end
|
|
local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
local jobKey = baseKey .. jobId
|
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
|
if shouldRemoveDeduplicationKey then
|
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
|
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId)
|
|
end
|
|
removeJobKeys(jobKey)
|
|
end
|
|
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
|
|
local start = timestamp - maxAge * 1000
|
|
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf", "LIMIT", 0, maxLimit)
|
|
for i, jobId in ipairs(jobIds) do
|
|
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
|
end
|
|
if #jobIds > 0 then
|
|
if #jobIds < maxLimit then
|
|
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
|
else
|
|
for from, to in batches(#jobIds, 7000) do
|
|
rcall("ZREM", targetSet, unpack(jobIds, from, to))
|
|
end
|
|
end
|
|
end
|
|
end
|
|
--[[
|
|
Functions to remove jobs by max count.
|
|
]]
|
|
-- Includes
|
|
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
|
local start = maxCount
|
|
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
|
for i, jobId in ipairs(jobIds) do
|
|
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
|
end
|
|
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
|
end
|
|
local function removeLock(jobKey, stalledKey, token, jobId)
|
|
if token ~= "0" then
|
|
local lockKey = jobKey .. ':lock'
|
|
local lockToken = rcall("GET", lockKey)
|
|
if lockToken == token then
|
|
rcall("DEL", lockKey)
|
|
rcall("SREM", stalledKey, jobId)
|
|
else
|
|
if lockToken then
|
|
-- Lock exists but token does not match
|
|
return -6
|
|
else
|
|
-- Lock is missing completely
|
|
return -2
|
|
end
|
|
end
|
|
end
|
|
return 0
|
|
end
|
|
--[[
|
|
Function to create a new job from stored dedup-next data
|
|
when a deduplicated job with keepLastIfActive finishes.
|
|
At most one next job is created per deduplication ID.
|
|
Multiple triggers while active overwrite the dedup-next data,
|
|
so only the latest data is used.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to get max events value or set by default 10000.
|
|
]]
|
|
local function getOrSetMaxEvents(metaKey)
|
|
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
|
if not maxEvents then
|
|
maxEvents = 10000
|
|
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
|
end
|
|
return maxEvents
|
|
end
|
|
--[[
|
|
Function to set the deduplication key for a job.
|
|
Uses TTL from deduplication opts if provided.
|
|
]]
|
|
local function setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
local ttl = deduplicationOpts and deduplicationOpts['ttl']
|
|
if ttl and ttl > 0 then
|
|
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
|
else
|
|
rcall('SET', deduplicationKey, jobId)
|
|
end
|
|
end
|
|
--[[
|
|
Shared helper to store a job and enqueue it into the appropriate list/set.
|
|
Handles delayed, prioritized, and standard (LIFO/FIFO) jobs.
|
|
Emits the appropriate event after enqueuing ("delayed" or "waiting").
|
|
Returns delay, priority from storeJob.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Adds a delayed job to the queue by doing the following:
|
|
- Creates a new job key with the job data.
|
|
- adds to delayed zset.
|
|
- Emits a global event 'delayed' if the job is delayed.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Bake in the job id first 12 bits into the timestamp
|
|
to guarantee correct execution order of delayed jobs
|
|
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
|
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
|
]]
|
|
local function getDelayedScore(delayedKey, timestamp, delay)
|
|
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
|
|
local minScore = delayedTimestamp * 0x1000
|
|
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
|
|
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
|
|
minScore, "WITHSCORES","LIMIT", 0, 1)
|
|
if #result then
|
|
local currentMaxScore = tonumber(result[2])
|
|
if currentMaxScore ~= nil then
|
|
if currentMaxScore >= maxScore then
|
|
return maxScore, delayedTimestamp
|
|
else
|
|
return currentMaxScore + 1, delayedTimestamp
|
|
end
|
|
end
|
|
end
|
|
return minScore, delayedTimestamp
|
|
end
|
|
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
|
maxEvents, markerKey, delay)
|
|
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
|
rcall("ZADD", delayedKey, score, jobId)
|
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
|
"jobId", jobId, "delay", delayedTimestamp)
|
|
-- mark that a delayed job is available
|
|
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
end
|
|
--[[
|
|
Function to store a job
|
|
]]
|
|
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
parentKey, parentData, repeatJobKey)
|
|
local jsonOpts = cjson.encode(opts)
|
|
local delay = opts['delay'] or 0
|
|
local priority = opts['priority'] or 0
|
|
local debounceId = opts['de'] and opts['de']['id']
|
|
local optionalValues = {}
|
|
if parentKey ~= nil then
|
|
table.insert(optionalValues, "parentKey")
|
|
table.insert(optionalValues, parentKey)
|
|
table.insert(optionalValues, "parent")
|
|
table.insert(optionalValues, parentData)
|
|
end
|
|
if repeatJobKey then
|
|
table.insert(optionalValues, "rjk")
|
|
table.insert(optionalValues, repeatJobKey)
|
|
end
|
|
if debounceId then
|
|
table.insert(optionalValues, "deid")
|
|
table.insert(optionalValues, debounceId)
|
|
end
|
|
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
|
"timestamp", timestamp, "delay", delay, "priority", priority,
|
|
unpack(optionalValues))
|
|
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
|
return delay, priority
|
|
end
|
|
local function storeAndEnqueueJob(eventsKey, jobIdKey, jobId, name, data, opts,
|
|
timestamp, parentKey, parentData, repeatJobKey, maxEvents,
|
|
waitKey, pausedKey, activeKey, metaKey, prioritizedKey,
|
|
priorityCounterKey, delayedKey, markerKey)
|
|
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
|
|
opts, timestamp, parentKey, parentData, repeatJobKey)
|
|
if delay ~= 0 and delayedKey then
|
|
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
|
|
else
|
|
local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, activeKey)
|
|
if priority > 0 then
|
|
addJobWithPriority(markerKey, prioritizedKey, priority, jobId,
|
|
priorityCounterKey, isPausedOrMaxed)
|
|
else
|
|
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
|
addJobInTargetList(waitKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
|
end
|
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
|
"jobId", jobId)
|
|
end
|
|
return delay, priority
|
|
end
|
|
local function requeueDeduplicatedJob(prefix, deduplicationId, eventStreamKey,
|
|
metaKey, activeKey, waitKey, pausedKey, markerKey, prioritizedKey,
|
|
priorityCounterKey, delayedKey, timestamp)
|
|
local deduplicationNextKey = prefix .. "dn:" .. deduplicationId
|
|
if rcall("EXISTS", deduplicationNextKey) == 1 then
|
|
local nextData = rcall("HMGET", deduplicationNextKey,
|
|
"name", "data", "opts", "pk", "pd", "pdk", "rjk", "jid")
|
|
-- Always increment the counter to keep it monotonic
|
|
local nextId = rcall("INCR", prefix .. "id") .. ""
|
|
local storedJobId = nextData[8] -- index 8 = "jid" (8th field in the HMGET call above)
|
|
local newJobId
|
|
if storedJobId then
|
|
newJobId = storedJobId
|
|
else
|
|
newJobId = nextId
|
|
end
|
|
local newJobIdKey = prefix .. newJobId
|
|
local newOpts = cjson.decode(nextData[3])
|
|
local deduplicationKey = prefix .. "de:" .. deduplicationId
|
|
local parentKey = nextData[4] or nil
|
|
local parentData = nextData[5] or nil
|
|
local parentDependenciesKey = nextData[6] or nil
|
|
local repeatJobKey = nextData[7] or nil
|
|
-- Set dedup key for the new job (without TTL when keepLastIfActive,
|
|
-- so the key outlives the job's active duration)
|
|
local deOpts = newOpts['de']
|
|
if deOpts and deOpts['keepLastIfActive'] then
|
|
rcall('SET', deduplicationKey, newJobId)
|
|
else
|
|
setDeduplicationKey(deduplicationKey, newJobId, deOpts)
|
|
end
|
|
-- Store and enqueue using the shared helper (handles priority/lifo/delayed)
|
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
storeAndEnqueueJob(eventStreamKey, newJobIdKey, newJobId, nextData[1], nextData[2],
|
|
newOpts, timestamp, parentKey, parentData, repeatJobKey, maxEvents,
|
|
waitKey, pausedKey, activeKey, metaKey, prioritizedKey,
|
|
priorityCounterKey, delayedKey, markerKey)
|
|
-- Register as child dependency if the job has a parent
|
|
if parentDependenciesKey then
|
|
rcall("SADD", parentDependenciesKey, newJobIdKey)
|
|
end
|
|
-- Only delete the dedup-next hash after the job is fully created,
|
|
-- so that if any step above errors, the data is not permanently lost.
|
|
rcall("DEL", deduplicationNextKey)
|
|
end
|
|
end
|
|
--[[
|
|
Function to trim events, default 10000.
|
|
]]
|
|
-- Includes
|
|
local function trimEvents(metaKey, eventStreamKey)
|
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
if maxEvents then
|
|
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
|
else
|
|
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
|
end
|
|
end
|
|
--[[
|
|
Validate and move or add dependencies to parent.
|
|
]]
|
|
-- Includes
|
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
|
parentId, jobIdKey, returnvalue, timestamp )
|
|
local processedSet = parentKey .. ":processed"
|
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
|
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
|
end
|
|
--[[
|
|
Function to update a bunch of fields in a job.
|
|
]]
|
|
local function updateJobFields(jobKey, msgpackedFields)
|
|
if msgpackedFields and #msgpackedFields > 0 then
|
|
local fieldsToUpdate = cmsgpack.unpack(msgpackedFields)
|
|
if fieldsToUpdate then
|
|
rcall("HMSET", jobKey, unpack(fieldsToUpdate))
|
|
end
|
|
end
|
|
end
|
|
local jobIdKey = KEYS[12]
|
|
if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
-- Make sure it does not have pending dependencies
|
|
-- It must happen before removing lock
|
|
if ARGV[5] == "completed" then
|
|
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
|
|
return -4
|
|
end
|
|
if rcall("ZCARD", jobIdKey .. ":unsuccessful") ~= 0 then
|
|
return -9
|
|
end
|
|
end
|
|
local opts = cmsgpack.unpack(ARGV[8])
|
|
local token = opts['token']
|
|
local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
|
|
if errorCode < 0 then
|
|
return errorCode
|
|
end
|
|
updateJobFields(jobIdKey, ARGV[9]);
|
|
local attempts = opts['attempts']
|
|
local maxMetricsSize = opts['maxMetricsSize']
|
|
local maxCount = opts['keepJobs']['count']
|
|
local maxAge = opts['keepJobs']['age']
|
|
local maxLimit = opts['keepJobs']['limit'] or 1000
|
|
local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
|
|
local parentKey = jobAttributes[1] or ""
|
|
local parentId = ""
|
|
local parentQueueKey = ""
|
|
if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed
|
|
local jsonDecodedParent = cjson.decode(jobAttributes[2])
|
|
parentId = jsonDecodedParent['id']
|
|
parentQueueKey = jsonDecodedParent['queueKey']
|
|
end
|
|
local jobId = ARGV[1]
|
|
local timestamp = ARGV[2]
|
|
-- Remove from active list (if not active we shall return error)
|
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
|
if (numRemovedElements < 1) then
|
|
return -3
|
|
end
|
|
local eventStreamKey = KEYS[4]
|
|
local metaKey = KEYS[9]
|
|
-- Trim events before emitting them to avoid trimming events emitted in this script
|
|
trimEvents(metaKey, eventStreamKey)
|
|
local prefix = ARGV[7]
|
|
removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
|
|
-- Check if there is requeue data for this dedup ID (keepLastIfActive mode)
|
|
if jobAttributes[3] then
|
|
requeueDeduplicatedJob(prefix, jobAttributes[3], eventStreamKey,
|
|
metaKey, KEYS[2], KEYS[1], KEYS[8], KEYS[14], KEYS[3], KEYS[10],
|
|
KEYS[7], timestamp)
|
|
end
|
|
-- If job has a parent we need to
|
|
-- 1) remove this job id from parents dependencies
|
|
-- 2) move the job Id to parent "processed" set
|
|
-- 3) push the results into parent "results" list
|
|
-- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
|
|
if parentId == "" and parentKey ~= "" then
|
|
parentId = getJobIdFromKey(parentKey)
|
|
parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
|
|
end
|
|
if parentId ~= "" then
|
|
if ARGV[5] == "completed" then
|
|
local dependenciesSet = parentKey .. ":dependencies"
|
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
|
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
|
|
timestamp)
|
|
end
|
|
else
|
|
moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp)
|
|
end
|
|
end
|
|
local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
|
|
-- Remove job?
|
|
if maxCount ~= 0 then
|
|
local targetSet = KEYS[11]
|
|
-- Add to complete/failed set
|
|
rcall("ZADD", targetSet, timestamp, jobId)
|
|
rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
|
if ARGV[5] == "failed" then
|
|
rcall("HDEL", jobIdKey, "defa")
|
|
end
|
|
-- Remove old jobs?
|
|
if maxAge ~= nil then
|
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
|
|
end
|
|
if maxCount ~= nil and maxCount > 0 then
|
|
removeJobsByMaxCount(maxCount, targetSet, prefix)
|
|
end
|
|
else
|
|
removeJobKeys(jobIdKey)
|
|
if parentKey ~= "" then
|
|
-- TODO: when a child is removed when finished, result or failure in parent
|
|
-- must not be deleted, those value references should be deleted when the parent
|
|
-- is deleted
|
|
removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
|
|
end
|
|
end
|
|
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
|
|
if ARGV[5] == "failed" then
|
|
if tonumber(attemptsMade) >= tonumber(attempts) then
|
|
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
|
|
attemptsMade)
|
|
end
|
|
end
|
|
-- Collect metrics
|
|
if maxMetricsSize ~= "" then
|
|
collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
|
|
end
|
|
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
|
|
-- and not rate limited.
|
|
if (ARGV[6] == "1") then
|
|
local result = fetchNextJob(KEYS[1], KEYS[2], KEYS[3], eventStreamKey,
|
|
KEYS[6], KEYS[7], KEYS[8], metaKey, KEYS[10], KEYS[14], prefix,
|
|
timestamp, opts)
|
|
if result then
|
|
return result
|
|
end
|
|
end
|
|
local waitLen = rcall("LLEN", KEYS[1])
|
|
if waitLen == 0 then
|
|
local activeLen = rcall("LLEN", KEYS[2])
|
|
if activeLen == 0 then
|
|
local prioritizedLen = rcall("ZCARD", KEYS[3])
|
|
if prioritizedLen == 0 then
|
|
rcall("XADD", eventStreamKey, "*", "event", "drained")
|
|
end
|
|
end
|
|
end
|
|
return 0
|
|
else
|
|
return -1
|
|
end
|
|
`;
|
|
export const moveToFinished = {
|
|
name: 'moveToFinished',
|
|
content,
|
|
keys: 14,
|
|
};
|