2048 lines
78 KiB
JavaScript
2048 lines
78 KiB
JavaScript
/**
|
|
* Includes all the scripts needed by the queue and jobs.
|
|
*/
|
|
'use strict';
|
|
import { EventEmitter } from 'events';
|
|
import { Packr } from 'msgpackr';
|
|
const packer = new Packr({
|
|
useRecords: false,
|
|
encodeUndefinedAsNil: true,
|
|
});
|
|
const pack = packer.pack;
|
|
import { array2obj, clientCommandMessageReg, errorObject, getParentKey, isEmpty, isRedisVersionLowerThan, objectToFlatArray, optsDecodeMap, optsEncodeMap, parseObjectValues, tryCatch, } from '../utils';
|
|
import { version as packageVersion } from '../version';
|
|
import { finishedErrors } from './finished-errors';
|
|
import { QueueKeys } from './queue-keys';
|
|
/**
|
|
* Upper bound on the number of forward backfill iterations the `getJobs` Lua
|
|
* script performs to replace skipped ids (missing job hashes) within a bounded
|
|
* range. It caps the work done per call so a range full of missing jobs cannot
|
|
* scan the whole state unboundedly.
|
|
*/
|
|
const GET_JOBS_MAX_BACKFILL_ITERATIONS = 5;
|
|
export class RedisQueueBackend extends EventEmitter {
|
|
constructor(connection, name, keys, toKey, opts, blockingConnection, ownsConnection = true) {
|
|
var _a;
|
|
super();
|
|
this.connection = connection;
|
|
this.name = name;
|
|
this.blockingConnection = blockingConnection;
|
|
this.ownsConnection = ownsConnection;
|
|
this.version = packageVersion;
|
|
this.redisPrefix = (_a = opts.prefix) !== null && _a !== void 0 ? _a : 'bull';
|
|
const self = this;
|
|
this.queue = {
|
|
keys,
|
|
toKey,
|
|
opts,
|
|
get closing() {
|
|
return self.closing;
|
|
},
|
|
get client() {
|
|
return self.connection.client;
|
|
},
|
|
get blockingClient() {
|
|
var _a;
|
|
return (_a = self.blockingConnection) === null || _a === void 0 ? void 0 : _a.client;
|
|
},
|
|
get redisVersion() {
|
|
return self.connection.redisVersion;
|
|
},
|
|
get databaseType() {
|
|
return self.connection.databaseType;
|
|
},
|
|
};
|
|
this.moveToFinishedKeys = [
|
|
keys.wait,
|
|
keys.active,
|
|
keys.prioritized,
|
|
keys.events,
|
|
keys.stalled,
|
|
keys.limiter,
|
|
keys.delayed,
|
|
keys.paused,
|
|
keys.meta,
|
|
keys.pc,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
];
|
|
if (this.ownsConnection) {
|
|
this.forwardConnectionEvents();
|
|
}
|
|
}
|
|
/**
|
|
* Returns a sibling backend bound to a different queue that shares this
|
|
* backend's connection(s). Used by {@link FlowProducer} to operate on the
|
|
* many queues that a flow may span over a single connection. The sibling
|
|
* does not own the connection, so its `close`/`disconnect` are no-ops.
|
|
*/
|
|
forQueue(queueName, prefix) {
|
|
const resolvedPrefix = prefix !== null && prefix !== void 0 ? prefix : this.redisPrefix;
|
|
const queueKeys = new QueueKeys(resolvedPrefix);
|
|
return new RedisQueueBackend(this.connection, queueName, queueKeys.getKeys(queueName), (type) => queueKeys.toKey(queueName, type), Object.assign(Object.assign({}, this.queue.opts), { prefix: resolvedPrefix }), this.blockingConnection, false);
|
|
}
|
|
/**
|
|
* The queue's fully-qualified name (`"<prefix>:<queue>"`). This is the
|
|
* cross-backend logical identifier (e.g. used as a flow parent reference).
|
|
*/
|
|
get qualifiedName() {
|
|
return `${this.redisPrefix}:${this.name}`;
|
|
}
|
|
/**
|
|
* The concrete Redis keys for this queue (wait, active, events, …).
|
|
*/
|
|
get keys() {
|
|
return this.queue.keys;
|
|
}
|
|
/**
|
|
* Builds a namespaced Redis sub-key of the given `type`
|
|
* (`"<prefix>:<queue>:<type>"`).
|
|
*/
|
|
toKey(type) {
|
|
return this.queue.toKey(type);
|
|
}
|
|
/**
|
|
* Parses a Redis flow child key (`"<prefix>:<queue>:<id>"`) into its
|
|
* components. Inverse of {@link toKey}.
|
|
*/
|
|
parseNodeKey(key) {
|
|
const lastColon = key.lastIndexOf(':');
|
|
const prevColon = key.lastIndexOf(':', lastColon - 1);
|
|
if (lastColon === -1 || prevColon === -1) {
|
|
const [prefix = '', queueName = '', id = ''] = key.split(':');
|
|
return { prefix, queueName, id };
|
|
}
|
|
const prefix = key.slice(0, prevColon);
|
|
const queueName = key.slice(prevColon + 1, lastColon);
|
|
const id = key.slice(lastColon + 1);
|
|
return { prefix, queueName, id };
|
|
}
|
|
/**
|
|
* Builds the Redis client name (`"<prefix>:<base64(queue)><suffix>"`), used
|
|
* for `CLIENT SETNAME` and worker/queue discovery via `CLIENT LIST`.
|
|
*/
|
|
clientName(suffix = '') {
|
|
const base64Name = Buffer.from(this.name).toString('base64');
|
|
return `${this.redisPrefix}:${base64Name}${suffix}`;
|
|
}
|
|
/**
|
|
* Normalizes the events of the owned connection(s) into the backend's own
|
|
* `'ready' | 'error' | 'close'` events.
|
|
*/
|
|
forwardConnectionEvents() {
|
|
this.connection.on('error', err => this.emit('error', err));
|
|
this.connection.on('ready', () => this.emit('ready'));
|
|
this.connection.on('close', () => this.emit('close'));
|
|
if (this.blockingConnection) {
|
|
this.blockingConnection.on('error', err => this.emit('error', err));
|
|
this.blockingConnection.on('ready', () => this.emit('ready'));
|
|
}
|
|
}
|
|
/**
|
|
* Resolves once the backend's underlying connection(s) are ready.
|
|
*/
|
|
async waitUntilReady() {
|
|
await this.connection.client;
|
|
if (this.blockingConnection) {
|
|
await this.blockingConnection.client;
|
|
}
|
|
}
|
|
/**
|
|
* Closes the backend and its underlying connection(s).
|
|
*
|
|
* The dedicated blocking connection (if any) is closed first so that an
|
|
* in-flight blocking command (e.g. `bzpopmin`) is interrupted before the
|
|
* main connection is closed.
|
|
*/
|
|
async close(force = false) {
|
|
if (!this.ownsConnection) {
|
|
return;
|
|
}
|
|
if (!this.closing) {
|
|
this.closing = (async () => {
|
|
if (this.blockingConnection) {
|
|
await this.blockingConnection.close(force);
|
|
}
|
|
await this.connection.close(force);
|
|
})();
|
|
}
|
|
return this.closing;
|
|
}
|
|
/**
|
|
* Forcibly disconnects the backend's underlying connection(s).
|
|
*/
|
|
async disconnect() {
|
|
if (!this.ownsConnection) {
|
|
return;
|
|
}
|
|
await this.connection.disconnect();
|
|
if (this.blockingConnection) {
|
|
await this.blockingConnection.disconnect();
|
|
}
|
|
}
|
|
/**
|
|
* Sets a human-readable name on the underlying connection (CLIENT SETNAME).
|
|
* Unsupported-command and shutdown errors are swallowed.
|
|
*/
|
|
async setName(name) {
|
|
const client = await this.connection.client;
|
|
try {
|
|
await client.clientSetName(name);
|
|
}
|
|
catch (err) {
|
|
if (!clientCommandMessageReg.test(err.message) &&
|
|
!this.closing) {
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* The raw Redis client. Redis-specific escape hatch (used e.g. by
|
|
* `Queue.client`); not part of {@link IQueueBackend}.
|
|
*/
|
|
get client() {
|
|
return this.connection.client;
|
|
}
|
|
/**
|
|
* The raw blocking Redis client (a dedicated connection used for the
|
|
* blocking `waitForJob` primitive), if this backend was created with one.
|
|
* Redis-specific escape hatch; not part of {@link IQueueBackend}.
|
|
*/
|
|
get blockingClient() {
|
|
var _a;
|
|
return (_a = this.blockingConnection) === null || _a === void 0 ? void 0 : _a.client;
|
|
}
|
|
/**
|
|
* The detected Redis server version. Redis-specific escape hatch; not part
|
|
* of {@link IQueueBackend}.
|
|
*/
|
|
get redisVersion() {
|
|
return this.connection.redisVersion;
|
|
}
|
|
/**
|
|
* The detected datastore flavour (`redis`, `dragonfly`, `valkey`, …).
|
|
* Redis-specific escape hatch; not part of {@link IQueueBackend}.
|
|
*/
|
|
get databaseType() {
|
|
return this.connection.databaseType;
|
|
}
|
|
/**
|
|
* Smallest meaningful block timeout (seconds) given the blocking
|
|
* connection's capabilities.
|
|
*/
|
|
get minimumBlockTimeout() {
|
|
var _a;
|
|
return ((_a = this.blockingConnection) !== null && _a !== void 0 ? _a : this.connection).capabilities
|
|
.canBlockFor1Ms
|
|
? 0.001
|
|
: 0.002;
|
|
}
|
|
/**
|
|
* Interrupts the in-flight blocking wait by disconnecting the dedicated
|
|
* blocking connection. No-op if there is none.
|
|
*/
|
|
async disconnectBlocking(wait = true) {
|
|
if (this.blockingConnection) {
|
|
await this.blockingConnection.disconnect(wait);
|
|
}
|
|
}
|
|
/**
|
|
* Re-establishes the dedicated blocking connection after an interrupt.
|
|
*/
|
|
async reconnectBlocking() {
|
|
if (this.blockingConnection) {
|
|
await this.blockingConnection.reconnect();
|
|
}
|
|
}
|
|
/**
|
|
* Executes a registered Lua script on the given Redis client, resolving the
|
|
* versioned command name (e.g. `addJob:<packageVersion>`) so the script
|
|
* belonging to the current BullMQ version is invoked.
|
|
*
|
|
* @param client - The Redis client or pipeline/transaction on which to run the command.
|
|
* @param commandName - The base name of the Lua script (without version suffix).
|
|
* @param args - Positional arguments forwarded to the Lua script (keys followed by argv).
|
|
* @returns The raw result produced by the Lua script.
|
|
*
|
|
* @private
|
|
*/
|
|
execCommand(client, commandName, args) {
|
|
const commandNameWithVersion = `${commandName}:${this.version}`;
|
|
return client.runCommand(commandNameWithVersion, args);
|
|
}
|
|
/**
|
|
* Checks whether a job with the given id is present in the provided queue
|
|
* state.
|
|
*/
|
|
async isJobInState(state, jobId) {
|
|
const client = await this.queue.client;
|
|
if (state === 'waiting') {
|
|
return ((await this.isJobInState('wait', jobId)) ||
|
|
(await this.isJobInState('paused', jobId)));
|
|
}
|
|
if (state === 'wait' || state === 'active' || state === 'paused') {
|
|
const listKey = this.queue.toKey(state);
|
|
let result;
|
|
if (isRedisVersionLowerThan(this.queue.redisVersion, '6.0.6', this.queue.databaseType)) {
|
|
result = await this.execCommand(client, 'isJobInList', [
|
|
listKey,
|
|
jobId,
|
|
]);
|
|
}
|
|
else {
|
|
result = await client.lpos(listKey, jobId);
|
|
}
|
|
return Number.isInteger(result);
|
|
}
|
|
else if (state === 'prioritized' ||
|
|
state === 'completed' ||
|
|
state === 'failed' ||
|
|
state === 'delayed' ||
|
|
state === 'waiting-children') {
|
|
const score = await client.zscore(this.queue.toKey(state), jobId);
|
|
return score !== null;
|
|
}
|
|
throw new Error(`Unknown job state: ${state}`);
|
|
}
|
|
addDelayedJobArgs(job, encodedOpts, args, keysMap = this.queue.keys) {
|
|
const queueKeys = keysMap;
|
|
const keys = [
|
|
queueKeys.marker,
|
|
queueKeys.meta,
|
|
queueKeys.id,
|
|
queueKeys.delayed,
|
|
queueKeys.completed,
|
|
queueKeys.events,
|
|
];
|
|
keys.push(pack(args), job.data, encodedOpts);
|
|
return keys;
|
|
}
|
|
addDelayedJob(client, job, encodedOpts, args, keys = this.queue.keys) {
|
|
const argsList = this.addDelayedJobArgs(job, encodedOpts, args, keys);
|
|
return this.execCommand(client, 'addDelayedJob', argsList);
|
|
}
|
|
addPrioritizedJobArgs(job, encodedOpts, args, keysMap = this.queue.keys) {
|
|
const queueKeys = keysMap;
|
|
const keys = [
|
|
queueKeys.marker,
|
|
queueKeys.meta,
|
|
queueKeys.id,
|
|
queueKeys.prioritized,
|
|
queueKeys.delayed,
|
|
queueKeys.completed,
|
|
queueKeys.active,
|
|
queueKeys.events,
|
|
queueKeys.pc,
|
|
];
|
|
keys.push(pack(args), job.data, encodedOpts);
|
|
return keys;
|
|
}
|
|
addPrioritizedJob(client, job, encodedOpts, args, keys = this.queue.keys) {
|
|
const argsList = this.addPrioritizedJobArgs(job, encodedOpts, args, keys);
|
|
return this.execCommand(client, 'addPrioritizedJob', argsList);
|
|
}
|
|
addParentJobArgs(job, encodedOpts, args, keysMap = this.queue.keys) {
|
|
const queueKeys = keysMap;
|
|
const keys = [
|
|
queueKeys.meta,
|
|
queueKeys.id,
|
|
queueKeys.delayed,
|
|
queueKeys['waiting-children'],
|
|
queueKeys.completed,
|
|
queueKeys.events,
|
|
];
|
|
keys.push(pack(args), job.data, encodedOpts);
|
|
return keys;
|
|
}
|
|
addParentJob(client, job, encodedOpts, args, keys = this.queue.keys) {
|
|
const argsList = this.addParentJobArgs(job, encodedOpts, args, keys);
|
|
return this.execCommand(client, 'addParentJob', argsList);
|
|
}
|
|
addStandardJobArgs(job, encodedOpts, args, keysMap = this.queue.keys) {
|
|
const queueKeys = keysMap;
|
|
const keys = [
|
|
queueKeys.wait,
|
|
queueKeys.paused,
|
|
queueKeys.meta,
|
|
queueKeys.id,
|
|
queueKeys.completed,
|
|
queueKeys.delayed,
|
|
queueKeys.active,
|
|
queueKeys.events,
|
|
queueKeys.marker,
|
|
];
|
|
keys.push(pack(args), job.data, encodedOpts);
|
|
return keys;
|
|
}
|
|
addStandardJob(client, job, encodedOpts, args, keys = this.queue.keys) {
|
|
const argsList = this.addStandardJobArgs(job, encodedOpts, args, keys);
|
|
return this.execCommand(client, 'addStandardJob', argsList);
|
|
}
|
|
/**
|
|
* Low-level Redis adapter helper: queues/executes a single job insert on the
|
|
* provided client or transaction (pipeline/multi). This is the only place
|
|
* that needs a connection handle; the public {@link addJob} / {@link addJobs}
|
|
* operations obtain it from the backend itself.
|
|
*
|
|
* Kept public (but outside {@link IQueueBackend}) so that flow producers can
|
|
* batch inserts across queues onto a shared transaction.
|
|
*/
|
|
async addJobToTransaction(client, job, jobId, parentKeyOpts = {}, keys = this.queue.keys) {
|
|
const opts = job.opts;
|
|
const queueKeys = keys;
|
|
const parent = job.parent;
|
|
const args = [
|
|
queueKeys[''],
|
|
typeof jobId !== 'undefined' ? jobId : '',
|
|
job.name,
|
|
job.timestamp,
|
|
job.parentKey || null,
|
|
parentKeyOpts.parentDependenciesKey || null,
|
|
parent,
|
|
job.repeatJobKey,
|
|
job.deduplicationId ? `${queueKeys.de}:${job.deduplicationId}` : null,
|
|
];
|
|
const encodedOpts = pack(optsAsJSON(opts));
|
|
let result;
|
|
if (parentKeyOpts.addToWaitingChildren) {
|
|
result = await this.addParentJob(client, job, encodedOpts, args, keys);
|
|
}
|
|
else if (typeof opts.delay == 'number' && opts.delay > 0) {
|
|
result = await this.addDelayedJob(client, job, encodedOpts, args, keys);
|
|
}
|
|
else if (opts.priority) {
|
|
result = await this.addPrioritizedJob(client, job, encodedOpts, args, keys);
|
|
}
|
|
else {
|
|
result = await this.addStandardJob(client, job, encodedOpts, args, keys);
|
|
}
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
parentKey: parentKeyOpts.parentKey,
|
|
command: 'addJob',
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
async addJob(job, jobId, parentKeyOpts = {}) {
|
|
const client = await this.queue.client;
|
|
return this.addJobToTransaction(client, job, jobId, parentKeyOpts);
|
|
}
|
|
async addJobs(entries) {
|
|
const client = await this.queue.client;
|
|
const pipeline = client.pipeline();
|
|
// Queue each insert on the pipeline. The command is enqueued synchronously,
|
|
// so we do not need to await each call before executing the pipeline.
|
|
for (const entry of entries) {
|
|
this.addJobToTransaction(pipeline, entry.job, entry.jobId, entry.parentKeyOpts);
|
|
}
|
|
const results = (await pipeline.exec());
|
|
const ids = [];
|
|
for (const [err, id] of results) {
|
|
if (err) {
|
|
throw err;
|
|
}
|
|
ids.push(id);
|
|
}
|
|
return ids;
|
|
}
|
|
/**
|
|
* Atomically inserts a whole flow (tree) of jobs that may span multiple
|
|
* queues, returning one `[error, idOrCode]` tuple per entry in the same
|
|
* order they were provided. For the Redis adapter this is a single `MULTI`
|
|
* transaction; another backend would use a single SQL transaction.
|
|
*
|
|
* Each entry is self-describing (it carries its own queue `prefix` and
|
|
* `queueName`), so the operation does not need to be bound to a single
|
|
* queue's key map.
|
|
*/
|
|
async addFlow(entries) {
|
|
const client = await this.queue.client;
|
|
const multi = client.multi();
|
|
for (const entry of entries) {
|
|
const keys = new QueueKeys(entry.prefix).getKeys(entry.queueName);
|
|
await this.addJobToTransaction(multi, entry.jobData, entry.jobId, entry.parentKeyOpts, keys);
|
|
}
|
|
return (await multi.exec());
|
|
}
|
|
pauseArgs(pause, emitEvent = true) {
|
|
let src = 'wait', dst = 'paused';
|
|
if (!pause) {
|
|
src = 'paused';
|
|
dst = 'wait';
|
|
}
|
|
const keys = [src, dst, 'meta', 'prioritized'].map((name) => this.queue.toKey(name));
|
|
keys.push(this.queue.keys.events, this.queue.keys.delayed, this.queue.keys.marker);
|
|
const args = [pause ? 'paused' : 'resumed', emitEvent ? '1' : '0'];
|
|
return keys.concat(args);
|
|
}
|
|
async pause(pause) {
|
|
const client = await this.queue.client;
|
|
if (pause) {
|
|
const args = this.pauseArgs(true);
|
|
await this.execCommand(client, 'pause', args);
|
|
return;
|
|
}
|
|
let legacyPausedRemaining = 0;
|
|
let emitEvent = true;
|
|
do {
|
|
const args = this.pauseArgs(false, emitEvent);
|
|
legacyPausedRemaining = Number(await this.execCommand(client, 'pause', args));
|
|
emitEvent = false;
|
|
} while (legacyPausedRemaining > 0);
|
|
}
|
|
/**
|
|
* Removes a deduplication key from Redis so that a new job with the same
|
|
* deduplication id can be enqueued again. The key is only removed if it
|
|
* currently maps to the provided `jobId`, preventing races between
|
|
* producers and finishing jobs.
|
|
*
|
|
* @param deduplicationId - The deduplication id whose key should be cleared.
|
|
* @param jobId - The id of the job that currently owns the dedup key.
|
|
* @returns `1` if the key was removed, `0` otherwise.
|
|
*
|
|
* @private
|
|
*/
|
|
async removeDeduplicationKey(deduplicationId, jobId) {
|
|
const client = await this.queue.client;
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [`${queueKeys.de}:${deduplicationId}`];
|
|
const args = [jobId];
|
|
return this.execCommand(client, 'removeDeduplicationKey', keys.concat(args));
|
|
}
|
|
/**
|
|
* Registers a job scheduler and enqueues its next delayed iteration.
|
|
* The scheduler stores the template data/options so subsequent iterations
|
|
* can be produced automatically based on the repeat options.
|
|
*
|
|
* @param jobSchedulerId - The id that uniquely identifies this scheduler.
|
|
* @param nextMillis - Timestamp (ms since epoch) for the next iteration.
|
|
* @param templateData - Serialized template data reused for every iteration.
|
|
* @param templateOpts - Redis-encoded job options applied to every iteration.
|
|
* @param opts - Repeat options describing the scheduling pattern.
|
|
* @param delayedJobOpts - Options applied to the next delayed job that is produced.
|
|
* @param producerId - Optional id of the job that produced this iteration, used to prevent duplicates.
|
|
* @returns A tuple of `[jobId, delay]`, where `delay` is the computed delay in milliseconds
|
|
* for the next iteration. When `delay` is `0`, the job is enqueued immediately.
|
|
* @throws An error resolved from `finishedErrors` when the Lua script returns a negative status code.
|
|
*
|
|
* @private
|
|
*/
|
|
async addJobScheduler(jobSchedulerId, nextMillis, templateData, templateOpts, opts, delayedJobOpts,
|
|
// The job id of the job that produced this next iteration
|
|
producerId) {
|
|
const client = await this.queue.client;
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [
|
|
queueKeys.repeat,
|
|
queueKeys.delayed,
|
|
queueKeys.wait,
|
|
queueKeys.paused,
|
|
queueKeys.meta,
|
|
queueKeys.prioritized,
|
|
queueKeys.marker,
|
|
queueKeys.id,
|
|
queueKeys.events,
|
|
queueKeys.pc,
|
|
queueKeys.active,
|
|
];
|
|
const args = [
|
|
nextMillis,
|
|
pack(opts),
|
|
jobSchedulerId,
|
|
templateData,
|
|
pack(optsAsJSON(templateOpts)),
|
|
pack(optsAsJSON(delayedJobOpts)),
|
|
Date.now(),
|
|
queueKeys[''],
|
|
producerId ? this.queue.toKey(producerId) : '',
|
|
];
|
|
const result = await this.execCommand(client, 'addJobScheduler', keys.concat(args));
|
|
if (typeof result === 'number' && result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
command: 'addJobScheduler',
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
async updateJobSchedulerNextMillis(jobSchedulerId, nextMillis, templateData, delayedJobOpts,
|
|
// The job id of the job that produced this next iteration - TODO: remove in next breaking change
|
|
producerId) {
|
|
const client = await this.queue.client;
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [
|
|
queueKeys.repeat,
|
|
queueKeys.delayed,
|
|
queueKeys.wait,
|
|
queueKeys.paused,
|
|
queueKeys.meta,
|
|
queueKeys.prioritized,
|
|
queueKeys.marker,
|
|
queueKeys.id,
|
|
queueKeys.events,
|
|
queueKeys.pc,
|
|
producerId ? this.queue.toKey(producerId) : '',
|
|
queueKeys.active,
|
|
];
|
|
const args = [
|
|
nextMillis,
|
|
jobSchedulerId,
|
|
templateData,
|
|
pack(optsAsJSON(delayedJobOpts)),
|
|
Date.now(),
|
|
queueKeys[''],
|
|
producerId,
|
|
];
|
|
return this.execCommand(client, 'updateJobScheduler', keys.concat(args));
|
|
}
|
|
async removeJobScheduler(jobSchedulerId) {
|
|
const client = await this.queue.client;
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [queueKeys.repeat, queueKeys.delayed, queueKeys.events];
|
|
const args = [jobSchedulerId, queueKeys['']];
|
|
return this.execCommand(client, 'removeJobScheduler', keys.concat(args));
|
|
}
|
|
removeArgs(jobId, removeChildren) {
|
|
const keys = [jobId, 'repeat'].map(name => this.queue.toKey(name));
|
|
const args = [jobId, removeChildren ? 1 : 0, this.queue.toKey('')];
|
|
return keys.concat(args);
|
|
}
|
|
async remove(jobId, removeChildren) {
|
|
const client = await this.queue.client;
|
|
const args = this.removeArgs(jobId, removeChildren);
|
|
const result = await this.execCommand(client, 'removeJob', args);
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'removeJob',
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
async removeUnprocessedChildren(jobId) {
|
|
const client = await this.queue.client;
|
|
const args = [
|
|
this.queue.toKey(jobId),
|
|
this.queue.keys.meta,
|
|
this.queue.toKey(''),
|
|
jobId,
|
|
];
|
|
await this.execCommand(client, 'removeUnprocessedChildren', args);
|
|
}
|
|
async extendLock(jobId, token, duration, client) {
|
|
client = client || (await this.queue.client);
|
|
const args = [
|
|
this.queue.toKey(jobId) + ':lock',
|
|
this.queue.keys.stalled,
|
|
token,
|
|
duration,
|
|
jobId,
|
|
];
|
|
return this.execCommand(client, 'extendLock', args);
|
|
}
|
|
async extendLocks(jobIds, tokens, duration) {
|
|
const client = await this.queue.client;
|
|
const args = [
|
|
this.queue.keys.stalled,
|
|
this.queue.toKey(''),
|
|
pack(tokens),
|
|
pack(jobIds),
|
|
duration,
|
|
];
|
|
return this.execCommand(client, 'extendLocks', args);
|
|
}
|
|
async updateData(job, data) {
|
|
const client = await this.queue.client;
|
|
const keys = [this.queue.toKey(job.id)];
|
|
const dataJson = JSON.stringify(data);
|
|
const result = await this.execCommand(client, 'updateData', keys.concat([dataJson]));
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId: job.id,
|
|
command: 'updateData',
|
|
});
|
|
}
|
|
}
|
|
async updateProgress(jobId, progress) {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
this.queue.toKey(jobId),
|
|
this.queue.keys.events,
|
|
this.queue.keys.meta,
|
|
];
|
|
const progressJson = JSON.stringify(progress);
|
|
const result = await this.execCommand(client, 'updateProgress', keys.concat([jobId, progressJson]));
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'updateProgress',
|
|
});
|
|
}
|
|
}
|
|
async addLog(jobId, logRow, keepLogs) {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
this.queue.toKey(jobId),
|
|
this.queue.toKey(jobId) + ':logs',
|
|
];
|
|
const result = await this.execCommand(client, 'addLog', keys.concat([jobId, logRow, keepLogs ? keepLogs : '']));
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'addLog',
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
moveToFinishedArgs(job, val, propVal, shouldRemove, target, token, timestamp, fetchNext = true, fieldsToUpdate) {
|
|
var _a, _b, _c, _d, _e, _f, _g;
|
|
const queueKeys = this.queue.keys;
|
|
const opts = this.queue.opts;
|
|
const workerKeepJobs = target === 'completed' ? opts.removeOnComplete : opts.removeOnFail;
|
|
const metricsKey = this.queue.toKey(`metrics:${target}`);
|
|
const keys = this.moveToFinishedKeys;
|
|
keys[10] = queueKeys[target];
|
|
keys[11] = this.queue.toKey((_a = job.id) !== null && _a !== void 0 ? _a : '');
|
|
keys[12] = metricsKey;
|
|
keys[13] = this.queue.keys.marker;
|
|
const keepJobs = this.getKeepJobs(shouldRemove, workerKeepJobs);
|
|
const args = [
|
|
job.id,
|
|
timestamp,
|
|
propVal,
|
|
typeof val === 'undefined' ? 'null' : val,
|
|
target,
|
|
!fetchNext || this.queue.closing ? 0 : 1,
|
|
queueKeys[''],
|
|
pack({
|
|
token,
|
|
name: opts.name,
|
|
keepJobs,
|
|
limiter: opts.limiter,
|
|
lockDuration: opts.lockDuration,
|
|
attempts: job.opts.attempts,
|
|
maxMetricsSize: ((_b = opts.metrics) === null || _b === void 0 ? void 0 : _b.maxDataPoints)
|
|
? (_c = opts.metrics) === null || _c === void 0 ? void 0 : _c.maxDataPoints
|
|
: '',
|
|
fpof: !!((_d = job.opts) === null || _d === void 0 ? void 0 : _d.failParentOnFailure),
|
|
cpof: !!((_e = job.opts) === null || _e === void 0 ? void 0 : _e.continueParentOnFailure),
|
|
idof: !!((_f = job.opts) === null || _f === void 0 ? void 0 : _f.ignoreDependencyOnFailure),
|
|
rdof: !!((_g = job.opts) === null || _g === void 0 ? void 0 : _g.removeDependencyOnFailure),
|
|
}),
|
|
fieldsToUpdate ? pack(objectToFlatArray(fieldsToUpdate)) : void 0,
|
|
];
|
|
return keys.concat(args);
|
|
}
|
|
getKeepJobs(shouldRemove, workerKeepJobs) {
|
|
if (typeof shouldRemove === 'undefined') {
|
|
return workerKeepJobs || { count: shouldRemove ? 0 : -1 };
|
|
}
|
|
return typeof shouldRemove === 'object'
|
|
? shouldRemove
|
|
: typeof shouldRemove === 'number'
|
|
? { count: shouldRemove }
|
|
: { count: shouldRemove ? 0 : -1 };
|
|
}
|
|
async moveToFinished(jobId, args) {
|
|
const client = await this.queue.client;
|
|
const result = await this.execCommand(client, 'moveToFinished', args);
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'moveToFinished',
|
|
state: 'active',
|
|
});
|
|
}
|
|
else {
|
|
if (typeof result !== 'undefined') {
|
|
return raw2NextJobData(result);
|
|
}
|
|
}
|
|
}
|
|
drainArgs(delayed) {
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [
|
|
queueKeys.wait,
|
|
queueKeys.paused,
|
|
queueKeys.delayed,
|
|
queueKeys.prioritized,
|
|
queueKeys.repeat,
|
|
];
|
|
const args = [queueKeys[''], delayed ? '1' : '0'];
|
|
return keys.concat(args);
|
|
}
|
|
async drain(delayed) {
|
|
const client = await this.queue.client;
|
|
const args = this.drainArgs(delayed);
|
|
return this.execCommand(client, 'drain', args);
|
|
}
|
|
removeChildDependencyArgs(jobId, parentKey) {
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [queueKeys['']];
|
|
const args = [this.queue.toKey(jobId), parentKey];
|
|
return keys.concat(args);
|
|
}
|
|
async removeChildDependency(jobId, parentKey) {
|
|
const client = await this.queue.client;
|
|
const args = this.removeChildDependencyArgs(jobId, parentKey);
|
|
const result = await this.execCommand(client, 'removeChildDependency', args);
|
|
switch (result) {
|
|
case 0:
|
|
return true;
|
|
case 1:
|
|
return false;
|
|
default:
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
parentKey,
|
|
command: 'removeChildDependency',
|
|
});
|
|
}
|
|
}
|
|
getRangesArgs(types, start, end, asc) {
|
|
const queueKeys = this.queue.keys;
|
|
const transformedTypes = types.map(type => {
|
|
return type === 'waiting' ? 'wait' : type;
|
|
});
|
|
const keys = [queueKeys['']];
|
|
const args = [start, end, asc ? '1' : '0', ...transformedTypes];
|
|
return keys.concat(args);
|
|
}
|
|
async getRanges(types, start = 0, end = 1, asc = false) {
|
|
const client = await this.queue.client;
|
|
const args = this.getRangesArgs(types, start, end, asc);
|
|
return await this.execCommand(client, 'getRanges', args);
|
|
}
|
|
getJobsArgs(types, start, end, asc) {
|
|
const queueKeys = this.queue.keys;
|
|
const transformedTypes = [
|
|
...new Set(types.map(type => (type === 'waiting' ? 'wait' : type))),
|
|
];
|
|
const keys = [queueKeys['']];
|
|
const args = [
|
|
start,
|
|
end,
|
|
asc ? '1' : '0',
|
|
GET_JOBS_MAX_BACKFILL_ITERATIONS,
|
|
...transformedTypes,
|
|
];
|
|
return keys.concat(args);
|
|
}
|
|
/**
|
|
* Fetches job ids and their job hashes for the provided states in a single
|
|
* script, skipping ids whose job hash is missing (for example the deprecated
|
|
* wait list marker or jobs removed after their id was read). Each returned
|
|
* entry is a `[jobId, jobHashFields]` tuple grouped per requested type.
|
|
*/
|
|
async getJobs(types, start = 0, end = -1, asc = false) {
|
|
const client = await this.queue.client;
|
|
const args = this.getJobsArgs(types, start, end, asc);
|
|
return await this.execCommand(client, 'getJobs', args);
|
|
}
|
|
getCountsArgs(types) {
|
|
const queueKeys = this.queue.keys;
|
|
const transformedTypes = types.map(type => {
|
|
return type === 'waiting' ? 'wait' : type;
|
|
});
|
|
const keys = [queueKeys['']];
|
|
const args = [...transformedTypes];
|
|
return keys.concat(args);
|
|
}
|
|
async getCounts(types) {
|
|
const client = await this.queue.client;
|
|
const args = this.getCountsArgs(types);
|
|
return await this.execCommand(client, 'getCounts', args);
|
|
}
|
|
getCountsPerPriorityArgs(priorities) {
|
|
const keys = [
|
|
this.queue.keys.wait,
|
|
this.queue.keys.paused,
|
|
this.queue.keys.meta,
|
|
this.queue.keys.prioritized,
|
|
];
|
|
const args = priorities;
|
|
return keys.concat(args);
|
|
}
|
|
async getCountsPerPriority(priorities) {
|
|
const client = await this.queue.client;
|
|
const args = this.getCountsPerPriorityArgs(priorities);
|
|
return await this.execCommand(client, 'getCountsPerPriority', args);
|
|
}
|
|
getDependencyCountsArgs(jobId, types) {
|
|
const keys = [
|
|
`${jobId}:processed`,
|
|
`${jobId}:dependencies`,
|
|
`${jobId}:failed`,
|
|
`${jobId}:unsuccessful`,
|
|
].map(name => {
|
|
return this.queue.toKey(name);
|
|
});
|
|
const args = types;
|
|
return keys.concat(args);
|
|
}
|
|
async getDependencyCounts(jobId, types) {
|
|
const client = await this.queue.client;
|
|
const args = this.getDependencyCountsArgs(jobId, types);
|
|
return await this.execCommand(client, 'getDependencyCounts', args);
|
|
}
|
|
moveToCompletedArgs(job, returnvalue, removeOnComplete, token, fetchNext = false) {
|
|
const timestamp = Date.now();
|
|
return this.moveToFinishedArgs(job, returnvalue, 'returnvalue', removeOnComplete, 'completed', token, timestamp, fetchNext);
|
|
}
|
|
moveToFailedArgs(job, failedReason, removeOnFailed, token, fetchNext = false, fieldsToUpdate) {
|
|
const timestamp = Date.now();
|
|
return this.moveToFinishedArgs(job, failedReason, 'failedReason', removeOnFailed, 'failed', token, timestamp, fetchNext, fieldsToUpdate);
|
|
}
|
|
async isFinished(jobId, returnValue = false) {
|
|
const client = await this.queue.client;
|
|
const keys = ['completed', 'failed', jobId].map((key) => {
|
|
return this.queue.toKey(key);
|
|
});
|
|
return this.execCommand(client, 'isFinished', keys.concat([jobId, returnValue ? '1' : '']));
|
|
}
|
|
async getState(jobId) {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
'completed',
|
|
'failed',
|
|
'delayed',
|
|
'active',
|
|
'wait',
|
|
'paused',
|
|
'waiting-children',
|
|
'prioritized',
|
|
].map((key) => {
|
|
return this.queue.toKey(key);
|
|
});
|
|
if (isRedisVersionLowerThan(this.queue.redisVersion, '6.0.6', this.queue.databaseType)) {
|
|
return this.execCommand(client, 'getState', keys.concat([jobId]));
|
|
}
|
|
return this.execCommand(client, 'getStateV2', keys.concat([jobId]));
|
|
}
|
|
/**
|
|
* Change delay of a delayed job.
|
|
*
|
|
* Reschedules a delayed job by setting a new delay from the current time.
|
|
* For example, calling changeDelay(5000) will reschedule the job to execute
|
|
* 5000 milliseconds (5 seconds) from now, regardless of the original delay.
|
|
*
|
|
* @param jobId - the ID of the job to change the delay for.
|
|
* @param delay - milliseconds from now when the job should be processed.
|
|
* @returns delay in milliseconds.
|
|
* @throws JobNotExist
|
|
* This exception is thrown if jobId is missing.
|
|
* @throws JobNotInState
|
|
* This exception is thrown if job is not in delayed state.
|
|
*/
|
|
async changeDelay(jobId, delay) {
|
|
const client = await this.queue.client;
|
|
const args = this.changeDelayArgs(jobId, delay);
|
|
const result = await this.execCommand(client, 'changeDelay', args);
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'changeDelay',
|
|
state: 'delayed',
|
|
});
|
|
}
|
|
}
|
|
changeDelayArgs(jobId, delay) {
|
|
const timestamp = Date.now();
|
|
const keys = [
|
|
this.queue.keys.delayed,
|
|
this.queue.keys.meta,
|
|
this.queue.keys.marker,
|
|
this.queue.keys.events,
|
|
];
|
|
return keys.concat([
|
|
delay,
|
|
JSON.stringify(timestamp),
|
|
jobId,
|
|
this.queue.toKey(jobId),
|
|
]);
|
|
}
|
|
async changePriority(jobId, priority = 0, lifo = false) {
|
|
const client = await this.queue.client;
|
|
const args = this.changePriorityArgs(jobId, priority, lifo);
|
|
const result = await this.execCommand(client, 'changePriority', args);
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'changePriority',
|
|
});
|
|
}
|
|
}
|
|
changePriorityArgs(jobId, priority = 0, lifo = false) {
|
|
const keys = [
|
|
this.queue.keys.wait,
|
|
this.queue.keys.paused,
|
|
this.queue.keys.meta,
|
|
this.queue.keys.prioritized,
|
|
this.queue.keys.active,
|
|
this.queue.keys.pc,
|
|
this.queue.keys.marker,
|
|
];
|
|
return keys.concat([priority, this.queue.toKey(''), jobId, lifo ? 1 : 0]);
|
|
}
|
|
moveToDelayedArgs(jobId, timestamp, token, delay, opts = {}) {
|
|
const queueKeys = this.queue.keys;
|
|
const workerOpts = this.queue.opts;
|
|
const keys = [
|
|
queueKeys.marker,
|
|
queueKeys.active,
|
|
queueKeys.prioritized,
|
|
queueKeys.delayed,
|
|
this.queue.toKey(jobId),
|
|
queueKeys.events,
|
|
queueKeys.meta,
|
|
queueKeys.stalled,
|
|
queueKeys.wait,
|
|
queueKeys.limiter,
|
|
queueKeys.paused,
|
|
queueKeys.pc,
|
|
];
|
|
const fetchNext = opts.fetchNext && !this.queue.closing ? 1 : 0;
|
|
return keys.concat([
|
|
this.queue.keys[''],
|
|
timestamp,
|
|
jobId,
|
|
token,
|
|
delay,
|
|
opts.skipAttempt ? '1' : '0',
|
|
opts.fieldsToUpdate
|
|
? pack(objectToFlatArray(opts.fieldsToUpdate))
|
|
: void 0,
|
|
fetchNext,
|
|
fetchNext
|
|
? pack({
|
|
token,
|
|
lockDuration: workerOpts.lockDuration,
|
|
limiter: workerOpts.limiter,
|
|
name: workerOpts.name,
|
|
})
|
|
: void 0,
|
|
]);
|
|
}
|
|
moveToWaitingChildrenArgs(jobId, token, opts) {
|
|
const timestamp = Date.now();
|
|
const childKey = getParentKey(opts.child);
|
|
const keys = [
|
|
'active',
|
|
'waiting-children',
|
|
jobId,
|
|
`${jobId}:dependencies`,
|
|
`${jobId}:unsuccessful`,
|
|
'stalled',
|
|
'events',
|
|
].map(name => {
|
|
return this.queue.toKey(name);
|
|
});
|
|
return keys.concat([
|
|
token,
|
|
childKey !== null && childKey !== void 0 ? childKey : '',
|
|
JSON.stringify(timestamp),
|
|
jobId,
|
|
this.queue.toKey(''),
|
|
]);
|
|
}
|
|
isMaxedArgs() {
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [queueKeys.meta, queueKeys.active];
|
|
return keys;
|
|
}
|
|
async isMaxed() {
|
|
const client = await this.queue.client;
|
|
const args = this.isMaxedArgs();
|
|
return !!(await this.execCommand(client, 'isMaxed', args));
|
|
}
|
|
async moveToDelayed(jobId, timestamp, delay, token = '0', opts = {}) {
|
|
const client = await this.queue.client;
|
|
const args = this.moveToDelayedArgs(jobId, timestamp, token, delay, opts);
|
|
const result = await this.execCommand(client, 'moveToDelayed', args);
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'moveToDelayed',
|
|
state: 'active',
|
|
});
|
|
}
|
|
else if (typeof result !== 'undefined') {
|
|
return raw2NextJobData(result);
|
|
}
|
|
}
|
|
/**
|
|
* Move parent job to waiting-children state.
|
|
*
|
|
* @returns true if job is successfully moved, false if there are pending dependencies.
|
|
* @throws JobNotExist
|
|
* This exception is thrown if jobId is missing.
|
|
* @throws JobLockNotExist
|
|
* This exception is thrown if job lock is missing.
|
|
* @throws JobNotInState
|
|
* This exception is thrown if job is not in active state.
|
|
*/
|
|
async moveToWaitingChildren(jobId, token, opts = {}) {
|
|
const client = await this.queue.client;
|
|
const args = this.moveToWaitingChildrenArgs(jobId, token, opts);
|
|
const result = await this.execCommand(client, 'moveToWaitingChildren', args);
|
|
switch (result) {
|
|
case 0:
|
|
return true;
|
|
case 1:
|
|
return false;
|
|
default:
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'moveToWaitingChildren',
|
|
state: 'active',
|
|
});
|
|
}
|
|
}
|
|
getRateLimitTtlArgs(maxJobs) {
|
|
const keys = [
|
|
this.queue.keys.limiter,
|
|
this.queue.keys.meta,
|
|
];
|
|
return keys.concat([maxJobs !== null && maxJobs !== void 0 ? maxJobs : '0']);
|
|
}
|
|
async getRateLimitTtl(maxJobs) {
|
|
const client = await this.queue.client;
|
|
const args = this.getRateLimitTtlArgs(maxJobs);
|
|
return this.execCommand(client, 'getRateLimitTtl', args);
|
|
}
|
|
/**
|
|
* Remove jobs in a specific state.
|
|
*
|
|
* @returns Id jobs from the deleted records.
|
|
*/
|
|
async cleanJobsByState(state, timestamp, limit = 0) {
|
|
const client = await this.queue.client;
|
|
return this.execCommand(client, 'cleanJobsInSet', [
|
|
this.queue.toKey(state),
|
|
this.queue.toKey('events'),
|
|
this.queue.toKey('repeat'),
|
|
this.queue.toKey(''),
|
|
timestamp,
|
|
limit,
|
|
state,
|
|
]);
|
|
}
|
|
getJobSchedulerArgs(id) {
|
|
const keys = [this.queue.keys.repeat];
|
|
return keys.concat([id]);
|
|
}
|
|
async getJobScheduler(id) {
|
|
const client = await this.queue.client;
|
|
const args = this.getJobSchedulerArgs(id);
|
|
return this.execCommand(client, 'getJobScheduler', args);
|
|
}
|
|
async isJobScheduler(id) {
|
|
const client = await this.queue.client;
|
|
const exists = await client.hexists(`${this.queue.keys.repeat}:${id}`, 'ic');
|
|
return exists === 1;
|
|
}
|
|
async getJobSchedulerData(key) {
|
|
const client = await this.queue.client;
|
|
return client.hgetall(this.queue.toKey('repeat:' + key));
|
|
}
|
|
async getJobSchedulersRange(start, end, asc) {
|
|
const client = await this.queue.client;
|
|
const key = this.queue.keys.repeat;
|
|
return asc
|
|
? client.zrange(key, start, end, { WITHSCORES: true })
|
|
: client.zrevrange(key, start, end, { WITHSCORES: true });
|
|
}
|
|
async getJobSchedulersCount() {
|
|
const client = await this.queue.client;
|
|
return client.zcard(this.queue.keys.repeat);
|
|
}
|
|
retryJobArgs(jobId, lifo, token, opts = {}) {
|
|
const keys = [
|
|
this.queue.keys.active,
|
|
this.queue.keys.wait,
|
|
this.queue.keys.paused,
|
|
this.queue.toKey(jobId),
|
|
this.queue.keys.meta,
|
|
this.queue.keys.events,
|
|
this.queue.keys.delayed,
|
|
this.queue.keys.prioritized,
|
|
this.queue.keys.pc,
|
|
this.queue.keys.marker,
|
|
this.queue.keys.stalled,
|
|
];
|
|
const pushCmd = (lifo ? 'R' : 'L') + 'PUSH';
|
|
return keys.concat([
|
|
this.queue.toKey(''),
|
|
Date.now(),
|
|
pushCmd,
|
|
jobId,
|
|
token,
|
|
opts.fieldsToUpdate
|
|
? pack(objectToFlatArray(opts.fieldsToUpdate))
|
|
: void 0,
|
|
]);
|
|
}
|
|
async retryJob(jobId, lifo, token = '0', opts = {}) {
|
|
const client = await this.queue.client;
|
|
const args = this.retryJobArgs(jobId, lifo, token, opts);
|
|
const result = await this.execCommand(client, 'retryJob', args);
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'retryJob',
|
|
state: 'active',
|
|
});
|
|
}
|
|
}
|
|
moveJobsToWaitArgs(state, count, timestamp) {
|
|
const keys = [
|
|
this.queue.toKey(''),
|
|
this.queue.keys.events,
|
|
this.queue.toKey(state),
|
|
this.queue.toKey('wait'),
|
|
this.queue.toKey('paused'),
|
|
this.queue.keys.meta,
|
|
this.queue.keys.active,
|
|
this.queue.keys.marker,
|
|
];
|
|
const args = [count, timestamp, state];
|
|
return keys.concat(args);
|
|
}
|
|
async retryFinishedJobs(state = 'failed', count = 1000, timestamp = new Date().getTime()) {
|
|
const client = await this.queue.client;
|
|
const args = this.moveJobsToWaitArgs(state, count, timestamp);
|
|
return this.execCommand(client, 'moveJobsToWait', args);
|
|
}
|
|
async promoteJobs(count = 1000) {
|
|
const client = await this.queue.client;
|
|
const args = this.moveJobsToWaitArgs('delayed', count, Number.MAX_VALUE);
|
|
return this.execCommand(client, 'moveJobsToWait', args);
|
|
}
|
|
/**
|
|
* Attempts to reprocess a job
|
|
*
|
|
* @param job - The job to reprocess
|
|
* @param state - The expected job state. If the job is not found
|
|
* on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
|
|
*
|
|
* @returns A promise that resolves when the job has been successfully moved to the wait queue.
|
|
* @throws Will throw an error with a code property indicating the failure reason:
|
|
* - code 0: Job does not exist
|
|
* - code -1: Job is currently locked and can't be retried
|
|
* - code -2: Job was not found in the expected set
|
|
*/
|
|
async retryFinishedJob(job, state, opts = {}) {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
this.queue.toKey(job.id),
|
|
this.queue.keys.events,
|
|
this.queue.toKey(state),
|
|
this.queue.keys.wait,
|
|
this.queue.keys.meta,
|
|
this.queue.keys.paused,
|
|
this.queue.keys.active,
|
|
this.queue.keys.marker,
|
|
];
|
|
const args = [
|
|
job.id,
|
|
(job.opts.lifo ? 'R' : 'L') + 'PUSH',
|
|
state === 'failed' ? 'failedReason' : 'returnvalue',
|
|
state,
|
|
opts.resetAttemptsMade ? '1' : '0',
|
|
opts.resetAttemptsStarted ? '1' : '0',
|
|
];
|
|
const result = await this.execCommand(client, 'reprocessJob', keys.concat(args));
|
|
switch (result) {
|
|
case 1:
|
|
return;
|
|
default:
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId: job.id,
|
|
command: 'reprocessJob',
|
|
state,
|
|
});
|
|
}
|
|
}
|
|
async getMetrics(type, start = 0, end = -1) {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
this.queue.toKey(`metrics:${type}`),
|
|
this.queue.toKey(`metrics:${type}:data`),
|
|
];
|
|
const args = [start, end];
|
|
const result = await this.execCommand(client, 'getMetrics', keys.concat(args));
|
|
return result;
|
|
}
|
|
async getClientList() {
|
|
const client = await this.queue.client;
|
|
if (client.isCluster && typeof client.nodes === 'function') {
|
|
const clusterNodes = client.nodes() || [];
|
|
return Promise.all(clusterNodes.map((node) => typeof node.clientList === 'function'
|
|
? node.clientList()
|
|
: node.client('LIST')));
|
|
}
|
|
return [await client.clientList()];
|
|
}
|
|
async moveToActive(token, name) {
|
|
const client = await this.queue.client;
|
|
const opts = this.queue.opts;
|
|
const queueKeys = this.queue.keys;
|
|
const keys = [
|
|
queueKeys.wait,
|
|
queueKeys.active,
|
|
queueKeys.prioritized,
|
|
queueKeys.events,
|
|
queueKeys.stalled,
|
|
queueKeys.limiter,
|
|
queueKeys.delayed,
|
|
queueKeys.paused,
|
|
queueKeys.meta,
|
|
queueKeys.pc,
|
|
queueKeys.marker,
|
|
];
|
|
const args = [
|
|
queueKeys[''],
|
|
Date.now(),
|
|
pack({
|
|
token,
|
|
lockDuration: opts.lockDuration,
|
|
limiter: opts.limiter,
|
|
name,
|
|
}),
|
|
];
|
|
const result = await this.execCommand(client, 'moveToActive', keys.concat(args));
|
|
return raw2NextJobData(result);
|
|
}
|
|
async promote(jobId) {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
this.queue.keys.delayed,
|
|
this.queue.keys.wait,
|
|
this.queue.keys.paused,
|
|
this.queue.keys.meta,
|
|
this.queue.keys.prioritized,
|
|
this.queue.keys.active,
|
|
this.queue.keys.pc,
|
|
this.queue.keys.events,
|
|
this.queue.keys.marker,
|
|
];
|
|
const args = [this.queue.toKey(''), jobId];
|
|
const code = await this.execCommand(client, 'promote', keys.concat(args));
|
|
if (code < 0) {
|
|
throw this.finishedErrors({
|
|
code,
|
|
jobId,
|
|
command: 'promote',
|
|
state: 'delayed',
|
|
});
|
|
}
|
|
}
|
|
moveStalledJobsToWaitArgs() {
|
|
const opts = this.queue.opts;
|
|
const keys = [
|
|
this.queue.keys.stalled,
|
|
this.queue.keys.wait,
|
|
this.queue.keys.active,
|
|
this.queue.keys['stalled-check'],
|
|
this.queue.keys.meta,
|
|
this.queue.keys.paused,
|
|
this.queue.keys.marker,
|
|
this.queue.keys.events,
|
|
this.queue.keys.repeat,
|
|
];
|
|
const args = [
|
|
opts.maxStalledCount,
|
|
this.queue.toKey(''),
|
|
Date.now(),
|
|
opts.stalledInterval,
|
|
];
|
|
return keys.concat(args);
|
|
}
|
|
/**
|
|
* Looks for unlocked jobs in the active queue.
|
|
*
|
|
* The job was being worked on, but the worker process died and it failed to renew the lock.
|
|
* We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
|
|
* back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
|
|
* (e.g. if the job handler keeps crashing),
|
|
* we limit the number stalled job recoveries to settings.maxStalledCount.
|
|
*/
|
|
async moveStalledJobsToWait() {
|
|
const client = await this.queue.client;
|
|
const args = this.moveStalledJobsToWaitArgs();
|
|
return this.execCommand(client, 'moveStalledJobsToWait', args);
|
|
}
|
|
/**
|
|
* Moves a job back from Active to Wait.
|
|
* This script is used when a job has been manually rate limited and needs
|
|
* to be moved back to wait from active status.
|
|
*
|
|
* @param client - Redis client
|
|
* @param jobId - Job id
|
|
* @returns
|
|
*/
|
|
async moveJobFromActiveToWait(jobId, token = '0') {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
this.queue.keys.active,
|
|
this.queue.keys.wait,
|
|
this.queue.keys.stalled,
|
|
this.queue.keys.paused,
|
|
this.queue.keys.meta,
|
|
this.queue.keys.limiter,
|
|
this.queue.keys.prioritized,
|
|
this.queue.keys.marker,
|
|
this.queue.keys.events,
|
|
];
|
|
const args = [jobId, token, this.queue.toKey(jobId)];
|
|
const result = await this.execCommand(client, 'moveJobFromActiveToWait', keys.concat(args));
|
|
if (result < 0) {
|
|
throw this.finishedErrors({
|
|
code: result,
|
|
jobId,
|
|
command: 'moveJobFromActiveToWait',
|
|
state: 'active',
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
async obliterate(opts) {
|
|
const client = await this.queue.client;
|
|
const keys = [
|
|
this.queue.keys.meta,
|
|
this.queue.toKey(''),
|
|
];
|
|
const args = [opts.count, opts.force ? 'force' : null];
|
|
const result = await this.execCommand(client, 'obliterate', keys.concat(args));
|
|
if (result < 0) {
|
|
switch (result) {
|
|
case -1:
|
|
throw new Error('Cannot obliterate non-paused queue');
|
|
case -2:
|
|
throw new Error('Cannot obliterate queue with active jobs');
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
/**
|
|
* Paginate a set or hash keys.
|
|
* @param opts - options to define the pagination behaviour
|
|
*
|
|
*/
|
|
async paginate(key, opts) {
|
|
const client = await this.queue.client;
|
|
const keys = [key];
|
|
const maxIterations = 5;
|
|
const pageSize = opts.end >= 0 ? opts.end - opts.start + 1 : Infinity;
|
|
let cursor = '0', offset = 0, items, total, rawJobs, page = [], jobs = [];
|
|
do {
|
|
const args = [
|
|
opts.start + page.length,
|
|
opts.end,
|
|
cursor,
|
|
offset,
|
|
maxIterations,
|
|
];
|
|
if (opts.fetchJobs) {
|
|
args.push(1);
|
|
}
|
|
[cursor, offset, items, total, rawJobs] = await this.execCommand(client, 'paginate', keys.concat(args));
|
|
page = page.concat(items);
|
|
if (rawJobs && rawJobs.length) {
|
|
jobs = jobs.concat(rawJobs.map((rawJob) => rawToJobJson(array2obj(rawJob))));
|
|
}
|
|
// Important to keep this coercive inequality (!=) instead of strict inequality (!==)
|
|
} while (cursor != '0' && page.length < pageSize);
|
|
// If we get an array of arrays, it means we are paginating a hash
|
|
if (page.length && Array.isArray(page[0])) {
|
|
const result = [];
|
|
for (let index = 0; index < page.length; index++) {
|
|
const [id, value] = page[index];
|
|
try {
|
|
result.push({ id, v: JSON.parse(value) });
|
|
}
|
|
catch (err) {
|
|
result.push({ id, err: err.message });
|
|
}
|
|
}
|
|
return {
|
|
cursor,
|
|
items: result,
|
|
total,
|
|
jobs,
|
|
};
|
|
}
|
|
else {
|
|
return {
|
|
cursor,
|
|
items: page.map(item => ({ id: item })),
|
|
total,
|
|
jobs,
|
|
};
|
|
}
|
|
}
|
|
finishedErrors({ code, jobId, parentKey, command, state, }) {
|
|
return finishedErrors({ code, jobId, parentKey, command, state });
|
|
}
|
|
/**
|
|
* Low-level Redis adapter helper: atomically check-and-delete a single batch
|
|
* of candidate orphaned jobs. Driven by {@link removeOrphanedJobs}.
|
|
*/
|
|
async removeOrphanedJobsBatch(candidateJobIds, stateKeySuffixes, jobSubKeySuffixes) {
|
|
const client = await this.queue.client;
|
|
const args = [
|
|
this.queue.toKey(''),
|
|
stateKeySuffixes.length,
|
|
...stateKeySuffixes,
|
|
jobSubKeySuffixes.length,
|
|
...jobSubKeySuffixes,
|
|
...candidateJobIds,
|
|
];
|
|
return this.execCommand(client, 'removeOrphanedJobs', args);
|
|
}
|
|
async removeOrphanedJobs(count = 1000, limit = 0) {
|
|
const client = await this.queue.client;
|
|
const keys = this.queue.keys;
|
|
// Derive infrastructure suffixes dynamically from the queue key map
|
|
// so any future keys are automatically excluded without code changes.
|
|
const knownSuffixes = new Set(Object.keys(keys));
|
|
// State key suffixes (excluding '') — passed to the Lua script which
|
|
// uses TYPE to decide whether a key is a list / zset / set.
|
|
const stateKeySuffixes = Object.keys(keys).filter(s => s !== '');
|
|
// Known job sub-key suffixes (cleaned up during deletion).
|
|
const jobSubKeySuffixes = [
|
|
'logs',
|
|
'dependencies',
|
|
'processed',
|
|
'failed',
|
|
'unsuccessful',
|
|
'lock',
|
|
];
|
|
const basePrefix = keys[''];
|
|
const scanPattern = basePrefix + '*';
|
|
let totalRemoved = 0;
|
|
let cursor = '0';
|
|
do {
|
|
const [nextCursor, scannedKeys] = await client.scan(cursor, {
|
|
MATCH: scanPattern,
|
|
COUNT: count,
|
|
});
|
|
cursor = nextCursor;
|
|
// Extract unique potential job IDs from this batch.
|
|
const candidateJobIds = new Set();
|
|
for (const key of scannedKeys) {
|
|
const suffix = key.slice(basePrefix.length);
|
|
// Skip infrastructure keys (derived from the key map).
|
|
if (knownSuffixes.has(suffix)) {
|
|
continue;
|
|
}
|
|
// Skip sub-keys of infrastructure prefixes (e.g. repeat:xxx, de:xxx).
|
|
const colonIdx = suffix.indexOf(':');
|
|
if (colonIdx !== -1) {
|
|
const prefixPart = suffix.slice(0, colonIdx);
|
|
if (knownSuffixes.has(prefixPart)) {
|
|
continue;
|
|
}
|
|
}
|
|
// Extract the job ID portion (before first colon, or the whole suffix).
|
|
const jobId = colonIdx === -1 ? suffix : suffix.slice(0, colonIdx);
|
|
// For sub-keys, only consider known job sub-key suffixes.
|
|
if (colonIdx !== -1) {
|
|
const subKey = suffix.slice(colonIdx + 1);
|
|
if (!jobSubKeySuffixes.includes(subKey)) {
|
|
continue;
|
|
}
|
|
}
|
|
candidateJobIds.add(jobId);
|
|
}
|
|
if (candidateJobIds.size === 0) {
|
|
continue;
|
|
}
|
|
// Run the Lua script atomically for this batch of candidates.
|
|
const result = await this.removeOrphanedJobsBatch([...candidateJobIds], stateKeySuffixes, jobSubKeySuffixes);
|
|
totalRemoved += result || 0;
|
|
if (limit > 0 && totalRemoved >= limit) {
|
|
break;
|
|
}
|
|
} while (cursor !== '0');
|
|
return totalRemoved;
|
|
}
|
|
// ============================================================
|
|
// High-level finished transitions (consolidate Lua arg-building + exec)
|
|
// ============================================================
|
|
async moveToCompleted(job, returnValue, removeOnComplete, token, fetchNext) {
|
|
const stringifiedReturnValue = tryCatch(JSON.stringify, JSON, [
|
|
returnValue,
|
|
]);
|
|
if (stringifiedReturnValue === errorObject) {
|
|
throw errorObject.value;
|
|
}
|
|
const args = this.moveToCompletedArgs(job, stringifiedReturnValue, removeOnComplete, token, fetchNext);
|
|
const result = await this.moveToFinished(job.id, args);
|
|
const finishedOn = args[this.moveToFinishedKeys.length + 1];
|
|
return { result, finishedOn };
|
|
}
|
|
async moveToFailed(job, failedReason, removeOnFail, token, fetchNext, fieldsToUpdate) {
|
|
const args = this.moveToFailedArgs(job, failedReason, removeOnFail, token, fetchNext, fieldsToUpdate);
|
|
const result = await this.moveToFinished(job.id, args);
|
|
const finishedOn = args[this.moveToFinishedKeys.length + 1];
|
|
return { result, finishedOn };
|
|
}
|
|
// ============================================================
|
|
// Promoted job getters (previously direct client calls in Job)
|
|
// ============================================================
|
|
async getJobData(jobId) {
|
|
const client = await this.queue.client;
|
|
const jobData = await client.hgetall(this.queue.toKey(jobId));
|
|
return isEmpty(jobData)
|
|
? undefined
|
|
: rawToJobJson(jobData);
|
|
}
|
|
async getDeduplicationJobId(deduplicationId) {
|
|
const client = await this.queue.client;
|
|
return client.get(`${this.queue.keys.de}:${deduplicationId}`);
|
|
}
|
|
async getJobLogs(jobId, start, end, asc) {
|
|
const client = await this.queue.client;
|
|
const multi = client.multi();
|
|
const logsKey = this.queue.toKey(jobId + ':logs');
|
|
if (asc) {
|
|
multi.lrange(logsKey, start, end);
|
|
}
|
|
else {
|
|
multi.lrange(logsKey, -(end + 1), -(start + 1));
|
|
}
|
|
multi.llen(logsKey);
|
|
const result = (await multi.exec());
|
|
if (!asc) {
|
|
result[0][1].reverse();
|
|
}
|
|
return {
|
|
logs: result[0][1],
|
|
count: result[1][1],
|
|
};
|
|
}
|
|
async clearLogs(jobId, keepLogs) {
|
|
const client = await this.queue.client;
|
|
const logsKey = this.queue.toKey(jobId) + ':logs';
|
|
if (keepLogs) {
|
|
await client.ltrim(logsKey, -keepLogs, -1);
|
|
}
|
|
else {
|
|
await client.del(logsKey);
|
|
}
|
|
}
|
|
async getProcessedChildrenValues(jobId) {
|
|
const client = await this.queue.client;
|
|
return (await client.hgetall(this.queue.toKey(`${jobId}:processed`)));
|
|
}
|
|
async getIgnoredChildrenFailures(jobId) {
|
|
const client = await this.queue.client;
|
|
return client.hgetall(this.queue.toKey(`${jobId}:failed`));
|
|
}
|
|
async getDependencies(jobId, opts = {}) {
|
|
const client = await this.queue.client;
|
|
const multi = client.pipeline();
|
|
if (!opts.processed && !opts.unprocessed && !opts.ignored && !opts.failed) {
|
|
multi.hgetall(this.queue.toKey(`${jobId}:processed`));
|
|
multi.smembers(this.queue.toKey(`${jobId}:dependencies`));
|
|
multi.hgetall(this.queue.toKey(`${jobId}:failed`));
|
|
multi.zrange(this.queue.toKey(`${jobId}:unsuccessful`), 0, -1);
|
|
const [[err1, processed], [err2, unprocessed], [err3, ignored], [err4, failed],] = (await multi.exec());
|
|
return {
|
|
processed: parseObjectValues(processed),
|
|
unprocessed,
|
|
failed,
|
|
ignored,
|
|
};
|
|
}
|
|
else {
|
|
const defaultOpts = {
|
|
cursor: 0,
|
|
count: 20,
|
|
};
|
|
const childrenResultOrder = [];
|
|
if (opts.processed) {
|
|
childrenResultOrder.push('processed');
|
|
const processedOpts = Object.assign(Object.assign({}, defaultOpts), opts.processed);
|
|
multi.hscan(this.queue.toKey(`${jobId}:processed`), processedOpts.cursor, {
|
|
COUNT: processedOpts.count,
|
|
});
|
|
}
|
|
if (opts.unprocessed) {
|
|
childrenResultOrder.push('unprocessed');
|
|
const unprocessedOpts = Object.assign(Object.assign({}, defaultOpts), opts.unprocessed);
|
|
multi.sscan(this.queue.toKey(`${jobId}:dependencies`), unprocessedOpts.cursor, { COUNT: unprocessedOpts.count });
|
|
}
|
|
if (opts.ignored) {
|
|
childrenResultOrder.push('ignored');
|
|
const ignoredOpts = Object.assign(Object.assign({}, defaultOpts), opts.ignored);
|
|
multi.hscan(this.queue.toKey(`${jobId}:failed`), ignoredOpts.cursor, {
|
|
COUNT: ignoredOpts.count,
|
|
});
|
|
}
|
|
let failedCursor;
|
|
if (opts.failed) {
|
|
childrenResultOrder.push('failed');
|
|
const failedOpts = Object.assign(Object.assign({}, defaultOpts), opts.failed);
|
|
failedCursor = failedOpts.cursor + failedOpts.count;
|
|
multi.zrange(this.queue.toKey(`${jobId}:unsuccessful`), failedOpts.cursor, failedOpts.count - 1);
|
|
}
|
|
const results = (await multi.exec());
|
|
let processedCursor, processed, unprocessedCursor, unprocessed, failed, ignoredCursor, ignored;
|
|
childrenResultOrder.forEach((key, index) => {
|
|
switch (key) {
|
|
case 'processed': {
|
|
processedCursor = results[index][1][0];
|
|
const rawProcessed = results[index][1][1];
|
|
const transformedProcessed = {};
|
|
for (let ind = 0; ind < rawProcessed.length; ++ind) {
|
|
if (ind % 2) {
|
|
transformedProcessed[rawProcessed[ind - 1]] = JSON.parse(rawProcessed[ind]);
|
|
}
|
|
}
|
|
processed = transformedProcessed;
|
|
break;
|
|
}
|
|
case 'failed': {
|
|
failed = results[index][1];
|
|
break;
|
|
}
|
|
case 'ignored': {
|
|
ignoredCursor = results[index][1][0];
|
|
const rawIgnored = results[index][1][1];
|
|
const transformedIgnored = {};
|
|
for (let ind = 0; ind < rawIgnored.length; ++ind) {
|
|
if (ind % 2) {
|
|
transformedIgnored[rawIgnored[ind - 1]] = rawIgnored[ind];
|
|
}
|
|
}
|
|
ignored = transformedIgnored;
|
|
break;
|
|
}
|
|
case 'unprocessed': {
|
|
unprocessedCursor = results[index][1][0];
|
|
unprocessed = results[index][1][1];
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
return Object.assign(Object.assign(Object.assign(Object.assign({}, (processedCursor
|
|
? {
|
|
processed,
|
|
nextProcessedCursor: Number(processedCursor),
|
|
}
|
|
: {})), (ignoredCursor
|
|
? {
|
|
ignored,
|
|
nextIgnoredCursor: Number(ignoredCursor),
|
|
}
|
|
: {})), (failedCursor
|
|
? {
|
|
failed,
|
|
nextFailedCursor: failedCursor,
|
|
}
|
|
: {})), (unprocessedCursor
|
|
? { unprocessed, nextUnprocessedCursor: Number(unprocessedCursor) }
|
|
: {}));
|
|
}
|
|
}
|
|
// ============================================================
|
|
// Promoted queue metadata & maintenance keys (previously direct
|
|
// client calls in Queue / Worker)
|
|
// ============================================================
|
|
async setQueueMeta(values) {
|
|
const client = await this.queue.client;
|
|
return client.hset(this.queue.keys.meta, values);
|
|
}
|
|
async getQueueMetaField(field) {
|
|
const client = await this.queue.client;
|
|
return client.hget(this.queue.keys.meta, field);
|
|
}
|
|
async getQueueMetaFields(fields) {
|
|
const client = await this.queue.client;
|
|
return client.hmget(this.queue.keys.meta, ...fields);
|
|
}
|
|
async getQueueMeta() {
|
|
const client = await this.queue.client;
|
|
return client.hgetall(this.queue.keys.meta);
|
|
}
|
|
async removeQueueMetaFields(fields) {
|
|
const client = await this.queue.client;
|
|
return client.hdel(this.queue.keys.meta, ...fields);
|
|
}
|
|
async hasQueueMetaField(field) {
|
|
const client = await this.queue.client;
|
|
const exists = await client.hexists(this.queue.keys.meta, field);
|
|
return exists === 1;
|
|
}
|
|
async setRateLimit(expireTimeMs) {
|
|
const client = await this.queue.client;
|
|
await client.set(this.queue.keys.limiter, Number.MAX_SAFE_INTEGER, {
|
|
PX: expireTimeMs,
|
|
});
|
|
}
|
|
async removeRateLimitKey() {
|
|
const client = await this.queue.client;
|
|
return client.del(this.queue.keys.limiter);
|
|
}
|
|
async removeDeprecatedPriorityKey() {
|
|
const client = await this.queue.client;
|
|
return client.del(this.queue.toKey('priority'));
|
|
}
|
|
async deleteDeduplicationKey(deduplicationId) {
|
|
const client = await this.queue.client;
|
|
return client.del(`${this.queue.keys.de}:${deduplicationId}`);
|
|
}
|
|
async trimEvents(maxLength) {
|
|
const client = await this.queue.client;
|
|
return client.xtrim(this.queue.keys.events, 'MAXLEN', maxLength, {
|
|
approximate: true,
|
|
});
|
|
}
|
|
// ============================================================
|
|
// Worker blocking primitive (previously bzpopmin in Worker)
|
|
// ============================================================
|
|
async waitForJob(blockTimeout) {
|
|
var _a;
|
|
const conn = (_a = this.blockingConnection) !== null && _a !== void 0 ? _a : this.connection;
|
|
const bclient = (await this.queue.blockingClient);
|
|
const roundedTimeout = conn.capabilities.canDoubleTimeout
|
|
? blockTimeout
|
|
: Math.ceil(blockTimeout);
|
|
const bzpopmin = bclient.bzpopmin(this.queue.keys.marker, roundedTimeout);
|
|
// If the watchdog abandons this command below, its (possibly much later)
|
|
// rejection must not surface as an unhandled promise rejection.
|
|
bzpopmin.catch(() => null);
|
|
// We cannot trust that the blocking connection stays blocking for exactly
|
|
// the expected time due to issues in Redis and IORedis. In particular,
|
|
// blocking connections must use `maxRetriesPerRequest: null`, and with that
|
|
// setting IORedis silently re-queues and re-sends an interrupted blocking
|
|
// command after a reconnect instead of rejecting it (see #4479). As a
|
|
// result the awaited `bzpopmin` can hang forever after a transient
|
|
// connection reset, parking the worker's fetch loop permanently.
|
|
//
|
|
// To make this robust we race the blocking command against a watchdog
|
|
// timeout: when the deadline passes we disconnect the (owned) blocking
|
|
// connection to abandon the possibly-stuck command and resolve the wait as
|
|
// a timeout, so the worker loop always advances (and can promote due
|
|
// delayed jobs / write markers) regardless of whether IORedis ever settles
|
|
// the command.
|
|
let timedOut = false;
|
|
let watchdog;
|
|
const timeout = new Promise(resolve => {
|
|
watchdog = setTimeout(() => {
|
|
timedOut = true;
|
|
bclient.disconnect(false);
|
|
resolve(null);
|
|
}, roundedTimeout * 1000 + 1000);
|
|
});
|
|
try {
|
|
const result = await Promise.race([bzpopmin, timeout]);
|
|
if (result) {
|
|
const [, member, score] = result;
|
|
if (member) {
|
|
return { member, score: parseInt(score) };
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
finally {
|
|
clearTimeout(watchdog);
|
|
// The watchdog disconnected the blocking connection without letting
|
|
// IORedis auto-resend the abandoned command. Since we resolved the wait
|
|
// as a timeout (rather than surfacing a rejection to the worker's own
|
|
// reconnect path), re-establish the dedicated blocking connection here so
|
|
// the next `waitForJob` starts from a healthy, unblocked connection.
|
|
if (timedOut && !this.closing) {
|
|
try {
|
|
await this.reconnectBlocking();
|
|
}
|
|
catch (_b) {
|
|
// Ignored: the next waitForJob call will retry the reconnect.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
async publishEvent(fields, maxEvents) {
|
|
const client = await this.queue.client;
|
|
return client.xadd(this.queue.keys.events, '*', fields, {
|
|
MAXLEN: maxEvents,
|
|
approximate: true,
|
|
});
|
|
}
|
|
async readEvents(id, blockTimeout) {
|
|
const client = await this.queue.client;
|
|
const xread = client.xread([{ key: this.queue.keys.events, id }], {
|
|
BLOCK: blockTimeout,
|
|
});
|
|
// Redis XREAD `BLOCK 0` means "block forever". If `blockTimeout` is <= 0 we
|
|
// cannot enforce a bounded deadline, so fall back to the plain blocking read.
|
|
if (blockTimeout <= 0) {
|
|
return xread;
|
|
}
|
|
// If the watchdog abandons this command below, its (possibly much later)
|
|
// rejection must not surface as an unhandled promise rejection.
|
|
xread.catch(() => null);
|
|
// Same class of hang as `waitForJob` (#4479): the event-stream connection
|
|
// must use `maxRetriesPerRequest: null`, and with that setting IORedis
|
|
// silently re-queues and re-sends an interrupted blocking `XREAD` after a
|
|
// reconnect instead of rejecting it — so the awaited read can hang forever
|
|
// after a transient connection reset, permanently stalling the event
|
|
// consumer loop. Race the read against a watchdog: when the deadline passes
|
|
// (the server-side BLOCK plus a small margin) we disconnect the connection
|
|
// to abandon the stuck command and resolve as a timeout, so the consumer
|
|
// loop always advances and can re-issue the read.
|
|
let timedOut = false;
|
|
let watchdog;
|
|
const timeout = new Promise(resolve => {
|
|
watchdog = setTimeout(() => {
|
|
timedOut = true;
|
|
client.disconnect(false);
|
|
resolve(null);
|
|
}, blockTimeout + 1000);
|
|
});
|
|
try {
|
|
return (await Promise.race([xread, timeout]));
|
|
}
|
|
finally {
|
|
clearTimeout(watchdog);
|
|
// The watchdog disconnected the connection without letting IORedis
|
|
// auto-resend the abandoned command. Re-establish it here so the next
|
|
// read starts from a healthy, unblocked connection.
|
|
if (timedOut && !this.closing) {
|
|
try {
|
|
await this.connection.reconnect();
|
|
}
|
|
catch (_a) {
|
|
// Ignored: the next readEvents call will retry the reconnect.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
export function raw2NextJobData(raw) {
|
|
if (raw) {
|
|
const result = [null, raw[1], raw[2], raw[3]];
|
|
if (raw[0]) {
|
|
result[0] = rawToJobJson(array2obj(raw[0]));
|
|
}
|
|
return result;
|
|
}
|
|
return [];
|
|
}
|
|
/**
|
|
* Decodes the compact, stored job options ({@link RedisJobOptions}, short keys)
|
|
* back into their public form ({@link JobsOptions}). Internal to this backend.
|
|
*/
|
|
function optsFromJSON(rawOpts, optsDecode = optsDecodeMap) {
|
|
const opts = JSON.parse(rawOpts || '{}');
|
|
const optionEntries = Object.entries(opts);
|
|
const options = {};
|
|
for (const item of optionEntries) {
|
|
const [attributeName, value] = item;
|
|
if (optsDecode[attributeName]) {
|
|
options[optsDecode[attributeName]] =
|
|
value;
|
|
}
|
|
else {
|
|
if (attributeName === 'tm') {
|
|
options.telemetry = Object.assign(Object.assign({}, options.telemetry), { metadata: value });
|
|
}
|
|
else if (attributeName === 'omc') {
|
|
options.telemetry = Object.assign(Object.assign({}, options.telemetry), { omitContext: value });
|
|
}
|
|
else {
|
|
options[attributeName] = value;
|
|
}
|
|
}
|
|
}
|
|
return options;
|
|
}
|
|
/**
|
|
* Decodes a raw Redis job hash ({@link JobJsonRaw}) into the public, datastore
|
|
* agnostic representation ({@link JobJson}). This translates the abbreviated
|
|
* field names and numeric strings used in Redis back into their public
|
|
* counterparts. Internal to this backend.
|
|
*
|
|
* Note: `data`, `stacktrace`, `returnvalue` and `failedReason` are kept as
|
|
* their JSON-encoded string form, matching {@link Job.asJSON}.
|
|
*/
|
|
function rawToJobJson(raw) {
|
|
return {
|
|
id: raw.id,
|
|
name: raw.name,
|
|
data: raw.data || '{}',
|
|
opts: optsFromJSON(raw.opts),
|
|
progress: JSON.parse(raw.progress || '0'),
|
|
delay: parseInt(raw.delay),
|
|
priority: parseInt(raw.priority),
|
|
timestamp: parseInt(raw.timestamp),
|
|
attemptsStarted: parseInt(raw.ats || '0'),
|
|
attemptsMade: parseInt(raw.attemptsMade || raw.atm || '0'),
|
|
stalledCounter: parseInt(raw.stc || '0'),
|
|
finishedOn: raw.finishedOn ? parseInt(raw.finishedOn) : undefined,
|
|
processedOn: raw.processedOn ? parseInt(raw.processedOn) : undefined,
|
|
repeatJobKey: raw.rjk,
|
|
debounceId: raw.deid,
|
|
deduplicationId: raw.deid,
|
|
failedReason: raw.failedReason,
|
|
deferredFailure: raw.defa,
|
|
stacktrace: raw.stacktrace,
|
|
returnvalue: raw.returnvalue,
|
|
parentKey: raw.parentKey,
|
|
parent: raw.parent ? JSON.parse(raw.parent) : undefined,
|
|
processedBy: raw.pb,
|
|
};
|
|
}
|
|
/**
|
|
* Encodes public job options ({@link JobsOptions}) into their compact, stored
|
|
* form (short keys) before they are packed and persisted in Redis. Internal to
|
|
* this backend.
|
|
*/
|
|
function optsAsJSON(opts = {}, optsEncode = optsEncodeMap) {
|
|
const optionEntries = Object.entries(opts);
|
|
const options = {};
|
|
for (const [attributeName, value] of optionEntries) {
|
|
if (typeof value === 'undefined') {
|
|
continue;
|
|
}
|
|
if (attributeName in optsEncode) {
|
|
const compressableAttribute = attributeName;
|
|
const key = optsEncode[compressableAttribute];
|
|
options[key] = value;
|
|
}
|
|
else {
|
|
// Handle complex compressable fields separately
|
|
if (attributeName === 'telemetry') {
|
|
if (value.metadata !== undefined) {
|
|
options.tm = value.metadata;
|
|
}
|
|
if (value.omitContext !== undefined) {
|
|
options.omc = value.omitContext;
|
|
}
|
|
}
|
|
else {
|
|
options[attributeName] = value;
|
|
}
|
|
}
|
|
}
|
|
return options;
|
|
}
|